From b4c932d9fd8b33f49c679ed84146d99a6a2f9551 Mon Sep 17 00:00:00 2001 From: Bruno Andrade Date: Mon, 7 Sep 2026 21:24:50 -0300 Subject: [PATCH 1/6] ccoctl: Add signing key rotation foundation Define the provider-neutral command contract and shared orchestration for resumable bound service account signer rotation. Add strict JWKS validation, safe checkpoints and artifacts, cluster-wide rotation guards, publication reconciliation, and reboot intent. Provider adapters and command registration remain follow-up work. Assisted-by: GPT-5 Signed-off-by: Bruno Andrade --- docs/ccoctl-rotate-signing-key.md | 274 +++++ docs/ccoctl.md | 5 + docs/rotate-oidc-key.md | 192 +++- go.mod | 2 +- pkg/cmd/provisioning/jwks/jwks.go | 315 ++++++ pkg/cmd/provisioning/jwks/jwks_test.go | 328 ++++++ .../provisioning/rotation/artifact_store.go | 255 +++++ .../rotation/artifact_store_test.go | 438 ++++++++ .../provisioning/rotation/checkpoint_lock.go | 41 + .../rotation/checkpoint_lock_unix.go | 62 ++ .../rotation/checkpoint_lock_unix_test.go | 87 ++ .../rotation/checkpoint_lock_unsupported.go | 17 + .../provisioning/rotation/checkpoint_store.go | 541 ++++++++++ .../rotation/checkpoint_store_test.go | 736 ++++++++++++++ pkg/cmd/provisioning/rotation/command.go | 112 +++ pkg/cmd/provisioning/rotation/command_test.go | 308 ++++++ pkg/cmd/provisioning/rotation/contract.go | 634 ++++++++++++ .../provisioning/rotation/contract_test.go | 929 +++++++++++++++++ .../rotation/current_jwks_binding_test.go | 116 +++ .../current_publication_recheck_test.go | 117 +++ .../provisioning/rotation/jwks_artifacts.go | 50 + .../rotation/jwks_artifacts_test.go | 134 +++ pkg/cmd/provisioning/rotation/orchestrator.go | 938 ++++++++++++++++++ .../rotation/orchestrator_safety_test.go | 882 ++++++++++++++++ .../rotation/orchestrator_test.go | 813 +++++++++++++++ .../rotation/orchestrator_types.go | 282 ++++++ .../rotation/publication_overlap_test.go | 54 + .../provisioning/rotation/reboot_intent.go | 51 + .../provisioning/rotation/rotation_guard.go | 84 ++ .../rotation/rotation_guard_orchestrator.go | 234 +++++ .../rotation/rotation_guard_test.go | 290 ++++++ pkg/cmd/provisioning/rotation/signer_state.go | 246 +++++ .../rotation/signer_state_test.go | 545 ++++++++++ pkg/cmd/provisioning/rotation/workspace.go | 62 ++ .../provisioning/rotation/workspace_test.go | 92 ++ pkg/cmd/provisioning/utils.go | 58 +- pkg/cmd/provisioning/utils_test.go | 10 + 37 files changed, 10254 insertions(+), 80 deletions(-) create mode 100644 docs/ccoctl-rotate-signing-key.md create mode 100644 pkg/cmd/provisioning/jwks/jwks.go create mode 100644 pkg/cmd/provisioning/jwks/jwks_test.go create mode 100644 pkg/cmd/provisioning/rotation/artifact_store.go create mode 100644 pkg/cmd/provisioning/rotation/artifact_store_test.go create mode 100644 pkg/cmd/provisioning/rotation/checkpoint_lock.go create mode 100644 pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go create mode 100644 pkg/cmd/provisioning/rotation/checkpoint_lock_unix_test.go create mode 100644 pkg/cmd/provisioning/rotation/checkpoint_lock_unsupported.go create mode 100644 pkg/cmd/provisioning/rotation/checkpoint_store.go create mode 100644 pkg/cmd/provisioning/rotation/checkpoint_store_test.go create mode 100644 pkg/cmd/provisioning/rotation/command.go create mode 100644 pkg/cmd/provisioning/rotation/command_test.go create mode 100644 pkg/cmd/provisioning/rotation/contract.go create mode 100644 pkg/cmd/provisioning/rotation/contract_test.go create mode 100644 pkg/cmd/provisioning/rotation/current_jwks_binding_test.go create mode 100644 pkg/cmd/provisioning/rotation/current_publication_recheck_test.go create mode 100644 pkg/cmd/provisioning/rotation/jwks_artifacts.go create mode 100644 pkg/cmd/provisioning/rotation/jwks_artifacts_test.go create mode 100644 pkg/cmd/provisioning/rotation/orchestrator.go create mode 100644 pkg/cmd/provisioning/rotation/orchestrator_safety_test.go create mode 100644 pkg/cmd/provisioning/rotation/orchestrator_test.go create mode 100644 pkg/cmd/provisioning/rotation/orchestrator_types.go create mode 100644 pkg/cmd/provisioning/rotation/publication_overlap_test.go create mode 100644 pkg/cmd/provisioning/rotation/reboot_intent.go create mode 100644 pkg/cmd/provisioning/rotation/rotation_guard.go create mode 100644 pkg/cmd/provisioning/rotation/rotation_guard_orchestrator.go create mode 100644 pkg/cmd/provisioning/rotation/rotation_guard_test.go create mode 100644 pkg/cmd/provisioning/rotation/signer_state.go create mode 100644 pkg/cmd/provisioning/rotation/signer_state_test.go create mode 100644 pkg/cmd/provisioning/rotation/workspace.go create mode 100644 pkg/cmd/provisioning/rotation/workspace_test.go diff --git a/docs/ccoctl-rotate-signing-key.md b/docs/ccoctl-rotate-signing-key.md new file mode 100644 index 0000000000..50a794535b --- /dev/null +++ b/docs/ccoctl-rotate-signing-key.md @@ -0,0 +1,274 @@ +# `ccoctl` signing-key rotation command contract + +> **Status:** The shared phase engine and command factory implement this provider-neutral contract, +> but the provider commands and their concrete cluster/cloud adapters are not registered yet. Until +> those integrations are available, use the [manual rotation procedure](rotate-oidc-key.md). + +## Scope + +The command is invoked externally for exactly one provider and one cluster: + +```text +ccoctl aws rotate-signing-key [options] +ccoctl azure rotate-signing-key [options] +ccoctl gcp rotate-signing-key [options] +``` + +One invocation follows the existing rotation sequence: preserve the current JWKS, request a +replacement signer, publish overlapping old and new public keys, wait for the cluster and nodes to +stabilize, and only then retire the old public keys. + +This is an operator-initiated workflow. It does not add an in-cluster controller, autonomous +scheduler, recurring job, or cadence API. A desired annual cadence remains an operational decision +outside this command. + +## Common command options + +The existing `ccoctl` option names are retained where they already express the required input. The +following names are the proposed v1 common contract: + +```text +--kubeconfig +--output-dir +--publication-mode +--resume +``` + +- `--kubeconfig` explicitly selects the target cluster. There is no in-cluster configuration + fallback. +- `--output-dir` holds public JWKS artifacts and the non-secret checkpoint record. +- `--publication-mode` selects direct provider operations or a paused manual workflow. The command + must not silently switch modes. +- `--resume` continues the operation recorded in `--output-dir` after validating its state against + the cluster and provider. + +Provider resource selectors, cloud-authentication options, manual artifact-import options, and the +manual publication acknowledgment mechanism are not settled by CCO-862. They must be resolved by +the provider adapter stories instead of being inferred here. + +## Required inputs and access + +The command contract requires the following capabilities: + +- An explicit kubeconfig for the target cluster. The command must not fall back to in-cluster + configuration. +- Explicit external cloud authentication for the selected provider. Credentials may be used in + memory but must not be copied into artifacts or checkpoint state, and the command must not obtain + cloud credentials from cluster Secrets. +- Enough cluster permission to inspect stability, read the public-only + `openshift-kube-apiserver/bound-sa-token-signing-certs` ConfigMap, inspect only the metadata of + the `openshift-kube-apiserver-operator/next-bound-service-account-signing-key` Secret, delete that + Secret with UID and resource version preconditions to trigger regeneration, acquire and complete + the cluster-durable signer-rotation guard, and reboot and observe the applicable worker and + control-plane machine config pools. +- Enough cloud permission to locate and read the current JWKS and to publish both the combined and + final JWKS to the supported target. +- A durable working directory for public artifacts and resumable, non-secret checkpoint state. +- A direct-publication or manual-publication choice. Manual operation must support supplying the + current JWKS and exporting each publication artifact. It also requires positive confirmation that + each artifact was applied, but the acknowledgment interface remains unresolved. + +Only public signer material is an input or output of this workflow. Public signer bytes and the +derived JWKS key ID are read from the public-only +`openshift-kube-apiserver/bound-sa-token-signing-certs` ConfigMap. The signer Secret is accessed only +through metadata-only retrieval of its UID and resource version for delete preconditions; the +command must never perform a normal Secret GET that returns its data. The command must never +require, export, or persist the private service-account signing key. It must also avoid persisting +kubeconfig contents, cloud credentials, bearer tokens, Secret payloads, or other authentication +material. + +The cluster adapter must request only the `meta.k8s.io/v1` `PartialObjectMetadata` +representation when reading signer Secret metadata. It must reject an unsupported response or any +full-object fallback rather than allowing the process to receive Secret data. + +The initial implementation accepts canonical `ccoctl`-owned JWKS documents. Unsupported top-level +or per-key extension fields fail before mutation or publication so the command cannot silently drop +issuer metadata. Supporting additional externally managed JWKS fields requires an explicit provider +contract that preserves them during combined-key publication. + +The shared package does not register an executable provider command, build a concrete cluster +adapter, resolve provider targets, or call cloud APIs. The AWS, Azure, and GCP adapter stories own +those integrations. The shared engine instead enforces the phase order, artifact/checkpoint rules, +conditional-publication contract, and reconciliation boundaries through injected adapters. + +## Preconditions + +Before requesting a replacement key, the command must verify and record that: + +1. The kubeconfig identifies the intended cluster and the cluster is stable. +2. The selected provider and issuer target can be resolved without assuming a public issuer + endpoint. +3. Cluster and cloud permissions needed by the selected publication mode are available. +4. A stable metadata-Secret/public-ConfigMap/metadata-Secret observation records the exact signer + Secret UID and resource version plus the complete public signer ConfigMap UID, resource version, + entry names, public-value digests, and derived key IDs. +5. A deterministic guard reference has been derived from that evidence. Its scope serializes all + signer rotations for the cluster, while its operation ID also binds the provider and exact target. +6. The current JWKS has been retrieved, parsed, saved as a non-secret recovery artifact, and every + current key ID is present in the recorded public signer baseline. +7. The working directory contains no conflicting in-progress rotation, unless the caller is + explicitly resuming it. + +A failed preflight must not mutate the cluster or provider. + +## Artifacts and checkpoint state + +The workflow produces the same public JWKS artifacts used by the manual procedure: + +- `serviceaccount-signer.public`: the replacement public signer retrieved from the cluster. +- `jwks.current.json`: the validated provider state captured before cluster mutation. +- `jwks.new.json`: the replacement public key only. +- `jwks.combined.json`: the retained current keys plus the replacement public key. +- `rotation-state.json`: a machine-readable checkpoint record containing the provider, stable + cluster identity, non-secret target identity, the complete pre-rotation public signer baseline, + the original signer Secret's metadata-only UID and resource version, the deterministic + cluster-wide guard reference, the exact appended public entry and replacement Secret reference, + validated artifact SHA-256 digests, the immutable reboot intent and its machine config pool + targets and node boot-ID baselines once recorded, last confirmed checkpoint, exact-artifact + publication confirmations, and a non-secret error code when applicable. + +Files containing private keys or credentials are not rotation artifacts. Logs and checkpoint data +must not include their contents. + +Public artifacts are validated before they are written, synced through a temporary file, atomically +renamed, and only then referenced by a checkpoint. An existing artifact name is immutable: an exact +retry is accepted, but different content fails closed. `jwks.new.json` must exactly represent the +replacement public key, and `jwks.combined.json` must be the canonical ordered union of the saved +current keys and that replacement, with all supported key metadata preserved. + +Local state updates use a persistent advisory lock in the output directory so concurrent checkpoint +writers cannot lose an update. The shared orchestrator holds the same workspace lock across cluster +or provider actions as well as their following checkpoint write; per-file locking alone is not +sufficient to serialize external mutations. The local lock is not a cluster-wide safety mechanism. +A separate cluster-durable guard serializes all signer rotations for the cluster, is verified before +post-preflight cluster observations and effects, and remains held through the final new-only +provider publication. Identical workspaces may converge on the same operation ID; a different +operation in the same scope fails closed. + +Checkpoint schema version 1 fixes generated JWKS encoding to two-space-indented JSON with one final +newline. A future implementation that changes that canonical representation must retain version 1 +compatibility or increment the checkpoint schema. The output directory must be on a trusted local +filesystem and must not be placed beneath a path that another user can replace or redirect. +Schema version 1 is the first supported rotation checkpoint format; checkpoints produced by earlier +development snapshots are not resumable. + +## Ordered checkpoints + +The implementation must durably record these exact schema-version-1 phases in order: + +1. **`initialized`:** the operation identity and publication mode are recorded before preflight. +2. **`preflight-complete`:** cluster identity, stability, target resolution, permissions, the + complete named public signer baseline, the signer Secret's metadata-only UID and resource + version, deterministic cluster-wide guard reference, and working state are validated and + recorded before the guard is acquired. +3. **`guard-acquired`:** the exact operation owns the cluster-durable signer-rotation guard. An + identical operation may adopt the same guard, while a different operation in the cluster scope + is rejected. +4. **`current-jwks-read`:** `jwks.current.json` is strictly validated, bound to the signer baseline, + and its digest is recorded. +5. **`next-key-requested`:** deletion of + `openshift-kube-apiserver-operator/next-bound-service-account-signing-key` is confirmed using the + recorded UID and resource version as preconditions. +6. **`next-public-key-read`:** the replacement Secret has a different exact UID/resource-version + reference, every baseline ConfigMap entry remains byte-identical, exactly one new named public + entry is present, and `serviceaccount-signer.public` is validated and bound to that evidence. +7. **`new-jwks-built`:** `jwks.new.json` is validated and its digest is recorded. +8. **`combined-jwks-built`:** `jwks.combined.json` contains the retained current keys and the new + key. +9. **`combined-jwks-published`:** the provider confirms the direct write, or the caller explicitly + acknowledges applying the exact manual artifact. +10. **`signer-rollout-stable`:** the cluster reports stability after adopting the replacement + signer. +11. **`reboot-intent-recorded`:** a non-secret operation ID, the exact machine config pool targets, + and each target node's baseline boot ID are durably recorded before requesting a reboot. The + shared engine derives the operation ID deterministically from the cluster identity and + replacement key ID so retries using another working directory converge even when their observed + targets or boot-ID baselines differ. The first reboot request atomically creates a + cluster-durable canonical record containing its exact targets and baselines. Later workspaces + adopt that canonical record before advancing, and it remains observable for the lifetime of + resumable operation state. +12. **`nodes-rebooted`:** reboot completion is confirmed for every node recorded in the immutable + reboot intent. +13. **`post-reboot-stable`:** the cluster reports stability after the node reboots. +14. **`new-only-jwks-published`:** the provider confirms the direct write, or the caller explicitly + acknowledges applying `jwks.new.json`. +15. **`guard-release-recorded`:** final cluster and provider state was reconciled while the exact + guard was still held, and the durable intent to complete that guard is recorded. +16. **`complete`:** the exact guard operation is durably marked complete. A crash after that + external update is reconciled from the guard's per-operation terminal record rather than by + accepting a later owner's guard. + +The command must never publish the new-only JWKS before combined publication, signer stability, +node reboot completion, and post-reboot stability are all confirmed. + +## Direct and manual publication + +In direct mode, the provider adapter reads and writes the supported JWKS target with the caller's +external cloud authentication. Every write is conditional on the opaque revision from an exact +predecessor read. The shared engine records success only after an exact readback of the desired +artifact; unrelated state is a conflict and is never overwritten. Immediately before each +publication, the engine verifies that the exact cluster guard is still held and that the complete +recorded signer evidence is unchanged. A local checkpoint is not proof of publication. + +In manual mode, the command emits the exact public artifact required for the next provider action +and pauses. Resume requires explicit confirmation bound to the publication phase, artifact name, +and SHA-256 digest. The command must not infer that a copy or upload succeeded, and must not pass a +publication checkpoint merely because the artifact exists locally. The public provider-specific +flags used to supply that acknowledgement remain an adapter integration decision. + +The cluster-wide signer-rotation guard remains held while a manual operation is paused. Losing or +replacing that exact guard is a conflict; the command must not silently reacquire a different +operation and continue from the old artifacts. + +Manual mode must also allow the caller to fetch the current JWKS out of band and supply it before +the command performs the cluster mutation. This supports private or externally managed issuers +without requiring a publicly reachable JWKS endpoint. + +## Interruption, retry, and recovery + +- A retry uses the same working directory and operation state. It validates observable cluster and + provider state before skipping a completed checkpoint. +- Preflight evidence and the deterministic guard reference are checkpointed before guard + acquisition. Resume observes and, only when still absent, idempotently acquires that same + operation; it never adopts a different operation in the cluster-global signer scope. +- If interrupted after requesting the replacement key, resume must compare the public key observed + in the complete public signer ConfigMap with the exact named baseline. Every baseline entry must + remain unchanged, the ConfigMap UID must match with a newer resource version, exactly one new + entry with a unique digest and derived key ID must exist, and the replacement Secret must have a + different, stable UID/resource-version reference. The old Secret is deleted only with the + recorded metadata-only UID and resource version preconditions. Resume must not request another + key merely because the previous run ended. +- If a provider write has an unknown outcome, retry must read or otherwise reconcile the target + before writing again. +- The reboot intent must be checkpointed before the disruptive request. Resume must reconcile node + boot IDs against the cluster-canonical baselines and reuse the same operation ID; it must not + submit a second pool reboot merely because the previous run ended or its outcome is unknown. +- Reboot planning, observation, reconciliation, and waits are read-only. The reboot request is the + only operation allowed to create the canonical reboot record or trigger the disruptive action. +- Repeated execution must not duplicate keys in the combined JWKS, regress to an earlier artifact, + or repeat a disruptive cluster action without validation. +- The cluster-wide guard is retained through final new-only publication and reconciliation. Its + release is an observable, idempotent external effect with a durable per-operation completion + record, so a crash cannot make a resumed run confuse a later owner with this operation. +- Invalid, missing, mismatched, or out-of-order state fails closed and preserves available public + recovery artifacts. +- Recovery must keep the overlapping JWKS published until the stability and reboot checkpoints are + confirmed. It must not retire old keys as an automatic rollback action. + +## Provider boundaries that remain unresolved + +The shared command must not hide provider capabilities that have not been agreed: + +- **AWS custom issuers:** the standard S3 publication path is in scope. For custom issuer storage, + including the OCPSTRAT-3586 case, it remains unresolved whether the command invokes a publisher + hook or produces an artifact and waits for an explicit acknowledgement. Required resource and + hook-selection options are therefore provider follow-up decisions. +- **GCP embedded JWK:** the GCS bucket `keys.json` path is the defined baseline. Support for an + embedded `pool-jwk-file` target is included only after explicit agreement; otherwise it is + follow-on work. Publication-target resource options remain a provider follow-up decision. +- **Discovery document:** the current provider stories specify JWKS publication. Whether rotation + also updates `.well-known/openid-configuration` or `jwks_uri` remains unresolved and must not be + assumed. +- **Availability:** overlapping keys reduce authentication risk, but this contract does not promise + zero downtime. Implementations and user documentation must surface the disruptive phases and the + availability implications for single-node and other non-highly-available clusters. diff --git a/docs/ccoctl.md b/docs/ccoctl.md index 31378c0349..6540a667b7 100644 --- a/docs/ccoctl.md +++ b/docs/ccoctl.md @@ -2,6 +2,11 @@ The `ccoctl` tool provides various commands to assist with the creating and maintenance of cloud credentials from outside the cluster (necessary when CCO is put in "Manual" mode). +The planned provider-specific signing-key rotation commands are defined by the +[`rotate-signing-key` command contract](ccoctl-rotate-signing-key.md). Their shared engine exists, +but the provider adapters and commands are not registered yet; use the +[manual rotation procedure](rotate-oidc-key.md) in the meantime. + - [AWS](#aws) - [Global flags](#global-flags) - [Creating RSA keys](#creating-rsa-keys) diff --git a/docs/rotate-oidc-key.md b/docs/rotate-oidc-key.md index c38ba075ea..cfd8b5d9b7 100644 --- a/docs/rotate-oidc-key.md +++ b/docs/rotate-oidc-key.md @@ -3,6 +3,22 @@ ## Overview When OpenShift is configured to use temporary credentials (AZWI, STS, WIF) to authenticate with the cloud platform api, special care must be taken when rotating the bound service account signer keys in order to reduce authentication failures. This can be accomplished by adding the new public key to the existing issuer file immediately after the cluster generates it. Once the cluster has fully updated to the new key all other keys can be removed. +This page describes the current manual procedure. The provider-neutral workflow is defined in the +[`ccoctl rotate-signing-key` command contract](ccoctl-rotate-signing-key.md), but its provider +adapters and commands are not registered yet. + +This manual procedure assumes an exclusive maintenance window in which no other user, automation, +or controller deletes `next-bound-service-account-signing-key`. Stop if that exclusivity cannot be +guaranteed: the cumulative public signer ConfigMap does not identify which Secret generation +produced an entry, so concurrent rotations cannot be disambiguated safely by this procedure. + +The provider upload examples below are legacy unconditional writes. Ensure exclusive publisher +access, verify that the remote JWKS still equals the expected predecessor immediately before each +upload, and read it back to compare with the exact uploaded file afterward. Stop if either +comparison fails or if the provider cannot be protected from another writer. The provider adapters +described by the command contract must replace this operational assumption with conditional writes +and exact readback. + ## Process 1. Configure environment variables @@ -11,7 +27,8 @@ When OpenShift is configured to use temporary credentials (AZWI, STS, WIF) to au Common ```bash - TEMPDIR=$(mktemp -d) + set -euo pipefail + TEMPDIR="$(mktemp -d)" ``` AWS @@ -47,24 +64,151 @@ When OpenShift is configured to use temporary credentials (AZWI, STS, WIF) to au oc adm wait-for-stable-cluster --minimum-stable-period=5s ``` +1. Download and inspect the current keys.json from the cloud provider. + + Save this recovery artifact before triggering rotation. Confirm that it came from the intended + issuer and contains at least one key. + + AWS + ```bash + aws s3api get-object --bucket "${AWS_BUCKET}" --key keys.json "${TEMPDIR}/jwks.current.download.json" + ``` + + Azure + ```bash + az storage blob download --container-name "${AZURE_STORAGE_CONTAINER}" --account-name "${AZURE_STORAGE_ACCOUNT}" --name 'openid/v1/jwks' -f "${TEMPDIR}/jwks.current.download.json" + ``` + + GCP public-bucket + ```bash + gcloud storage cp "gs://${GCP_BUCKET}/keys.json" "${TEMPDIR}/jwks.current.download.json" + ``` + + GCP pool-jwk-file + ```bash + gcloud iam workload-identity-pools providers describe --format json --location global --workload-identity-pool "${CLUSTER_NAME}" "${CLUSTER_NAME}" \ + | jq -er '.oidc.jwksJson' > "${TEMPDIR}/jwks.current.download.json" + ``` + + Perform this basic structural check before continuing. The forthcoming command also performs + strict RSA key, key ID, algorithm, purpose, duplicate, and signer-baseline validation that is + not reproduced by this legacy shell procedure. + + ```bash + jq -e ' + def supported_key: + type == "object" + and ((keys_unsorted - ["alg", "e", "kid", "kty", "n", "use", "x5c", "x5t", "x5t#S256", "x5u"]) | length) == 0 + and .kty == "RSA" + and (.kid | type == "string" and length > 0) + and (.n | type == "string" and length > 0) + and (.e | type == "string" and length > 0) + and ((.alg // "RS256") == "RS256") + and ((.use // "sig") == "sig"); + type == "object" + and ((keys_unsorted - ["keys"]) | length) == 0 + and (.keys | type == "array" and length > 0) + and all(.keys[]; supported_key) + and (([.keys[].kid] | length) == ([.keys[].kid] | unique | length)) + ' "${TEMPDIR}/jwks.current.download.json" > /dev/null + + mv "${TEMPDIR}/jwks.current.download.json" "${TEMPDIR}/jwks.current.json" + ``` + 1. Trigger the kube-apiserver to create a new bound service account signing key. - Deleting the next-bound-service-account-signing-key secret will cause the kube-apserver to generate a new one. At this point, the kube-apiserver will start rolling out the new key. In order to reduce the risk of authentication failures, it is important to complete all steps up to and including ***Upload the combined keys file*** as quickly as possible. + Deleting the `next-bound-service-account-signing-key` Secret asks the operator to generate a + replacement. The operator first appends the replacement public key to the cumulative signer + ConfigMap and rolls that verifier state through kube-apiserver revisions before it promotes the + replacement signer. In order to reduce the risk of authentication failures, it is important to + complete all steps up to and including ***Upload the combined keys file*** as quickly as + possible. WARNING: The remaining steps may cause downtime for the cluster. + Immediately before triggering rotation, save the public signer set so that the replacement can + be identified without reading any Secret data. + + ```bash + oc -n openshift-kube-apiserver get configmap/bound-sa-token-signing-certs -o json \ + > "${TEMPDIR}/bound-sa-token-signing-certs.before.json.tmp" + + jq -e 'type == "object"' "${TEMPDIR}/bound-sa-token-signing-certs.before.json.tmp" > /dev/null + mv "${TEMPDIR}/bound-sa-token-signing-certs.before.json.tmp" \ + "${TEMPDIR}/bound-sa-token-signing-certs.before.json" + ``` + ```bash oc -n openshift-kube-apiserver-operator delete secrets/next-bound-service-account-signing-key ``` 1. Download the new bound service account signing key public key - Download the public key from the freshly generated next-bound-service-account-signing-key secret. We will use this key to generate keys.json files to upload to the oidc issuer. + Read the public-only signer ConfigMap and select the one public key that was not present in the + pre-rotation snapshot. Do not select a fixed or highest-numbered `service-account-NNN.pub` + entry: the ConfigMap is cumulative and its entry names do not identify the active or next key. ```bash - oc get -n openshift-kube-apiserver-operator secret/next-bound-service-account-signing-key -ojsonpath='{ .data.service-account\.pub }' | base64 -d > ${TEMPDIR}/serviceaccount-signer.public + oc -n openshift-kube-apiserver get configmap/bound-sa-token-signing-certs -o json \ + > "${TEMPDIR}/bound-sa-token-signing-certs.after.json.tmp" + + jq -e 'type == "object"' "${TEMPDIR}/bound-sa-token-signing-certs.after.json.tmp" > /dev/null + mv "${TEMPDIR}/bound-sa-token-signing-certs.after.json.tmp" \ + "${TEMPDIR}/bound-sa-token-signing-certs.after.json" + + jq -enr \ + --slurpfile before "${TEMPDIR}/bound-sa-token-signing-certs.before.json" \ + --slurpfile after "${TEMPDIR}/bound-sa-token-signing-certs.after.json" ' + def signer_map: + (.data // {}) as $data + | if ($data | type) != "object" then + error("signer ConfigMap data is not an object") + elif ([$data | keys[] + | select(test("^service-account-[0-9]+\\.pub$") | not)] | length) != 0 then + error("signer ConfigMap contains an unexpected data entry") + elif ([$data[] | select(type != "string")] | length) != 0 then + error("signer ConfigMap contains a non-string value") + else $data + end; + ($before[0] | signer_map) as $old + | ($after[0] | signer_map) as $new + | [$old | to_entries[] | . as $entry + | select($new[$entry.key] != $entry.value)] as $changed + | [$new | to_entries[] | . as $entry + | select(($old | has($entry.key)) | not)] as $added + | if ($before | length) != 1 or ($after | length) != 1 then + error("expected exactly one JSON object in each signer snapshot") + elif (($before[0].metadata.uid // "") | length) == 0 + or (($after[0].metadata.uid // "") | length) == 0 then + error("signer ConfigMap UID is missing") + elif (($before[0].metadata.resourceVersion // "") | length) == 0 + or (($after[0].metadata.resourceVersion // "") | length) == 0 then + error("signer ConfigMap resource version is missing") + elif $before[0].metadata.uid != $after[0].metadata.uid then + error("signer ConfigMap was replaced") + elif $before[0].metadata.resourceVersion == $after[0].metadata.resourceVersion then + error("signer ConfigMap has not changed") + elif ($old | length) == 0 then + error("pre-rotation signer set is empty") + elif ($changed | length) != 0 then + error("a pre-rotation signer entry changed or disappeared") + elif ($added | length) != 1 then + error("expected exactly one new signer public key, found \($added | length)") + elif ([$old[]] | index($added[0].value)) != null then + error("new signer entry repeats a pre-rotation public key") + else $added[0].value + end + ' > "${TEMPDIR}/serviceaccount-signer.public.tmp" + + test -s "${TEMPDIR}/serviceaccount-signer.public.tmp" + mv "${TEMPDIR}/serviceaccount-signer.public.tmp" \ + "${TEMPDIR}/serviceaccount-signer.public" ``` + If no new signer is found, wait for the operator to update the ConfigMap and repeat this read. + If an existing entry changed or disappeared, the ConfigMap was replaced, or more than one new + signer is found, stop: another rotation may be in progress and the replacement is ambiguous. + 1. Create a keys.json using the new public key Use the public key downloaded above to create a new keys.json file. We do this by taking advantage of the --dry-run option in order to only output files on disk, including the new keys.json file. The actual values of many of the parameters is not important as they do not affect the generation of a new key. @@ -90,34 +234,26 @@ When OpenShift is configured to use temporary credentials (AZWI, STS, WIF) to au cp ${TEMPDIR}/04-keys.json ${TEMPDIR}/jwks.new.json ``` -1. Download the current keys.json from the cloud provider. - - AWS - ```bash - aws s3api get-object --bucket ${AWS_BUCKET} --key keys.json ${TEMPDIR}/jwks.current.json - ``` - - Azure - ```bash - az storage blob download --container-name ${AZURE_STORAGE_CONTAINER} --account-name ${AZURE_STORAGE_ACCOUNT} --name 'openid/v1/jwks' -f ${TEMPDIR}/jwks.current.json - ``` - - GCP public-bucket - ```bash - gcloud storage cp gs://${GCP_BUCKET}/keys.json ${TEMPDIR}/jwks.current.json - ``` - - GCP pool-jwk-file - ```bash - gcloud iam workload-identity-pools providers describe --format json --location global --workload-identity-pool ${CLUSTER_NAME} ${CLUSTER_NAME} | jq -r ".oidc.jwksJson" > ${TEMPDIR}/jwks.current.json - ``` - 1. Combine the current and new keys - Combine the key(s) downloaded from the cloud provider with the new key. The resulting file will enable authentication for both the old and new keys during the transistion. + Combine the key(s) downloaded from the cloud provider with the new key. The resulting file will enable authentication for both the old and new keys during the transition. ```bash - jq -s '{ keys: map(.keys[])}' ${TEMPDIR}/jwks.current.json ${TEMPDIR}/jwks.new.json > ${TEMPDIR}/jwks.combined.json + jq -e -s ' + .[0] as $current + | .[1] as $replacement + | ($current.keys | map(.kid)) as $current_ids + | if ($replacement.keys | type) != "array" or ($replacement.keys | length) != 1 then + error("replacement JWKS must contain exactly one key") + elif ($current_ids | index($replacement.keys[0].kid)) != null then + error("replacement key ID is already present in the current JWKS") + else + {keys: ($current.keys + $replacement.keys)} + end + ' "${TEMPDIR}/jwks.current.json" "${TEMPDIR}/jwks.new.json" \ + > "${TEMPDIR}/jwks.combined.json.tmp" + + mv "${TEMPDIR}/jwks.combined.json.tmp" "${TEMPDIR}/jwks.combined.json" ``` 1. Upload the combined keys file diff --git a/go.mod b/go.mod index 49b7e9a06c..7af75bf62c 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/tidwall/gjson v1.18.0 golang.org/x/mod v0.36.0 golang.org/x/oauth2 v0.36.0 + golang.org/x/sys v0.46.0 golang.org/x/time v0.15.0 google.golang.org/api v0.288.0 google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 @@ -210,7 +211,6 @@ require ( golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.45.0 // indirect diff --git a/pkg/cmd/provisioning/jwks/jwks.go b/pkg/cmd/provisioning/jwks/jwks.go new file mode 100644 index 0000000000..660d8ddf6c --- /dev/null +++ b/pkg/cmd/provisioning/jwks/jwks.go @@ -0,0 +1,315 @@ +package jwks + +import ( + "bytes" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "strings" + "unicode" + + jose "github.com/go-jose/go-jose/v4" +) + +// Artifact is a validated, encoded public JWKS and the non-secret metadata +// needed to bind it to a rotation checkpoint. +type Artifact struct { + Data []byte + SHA256 string + KeyIDs []string +} + +// Inspect validates an existing JWKS while preserving its exact bytes for +// recovery and digest comparison. +func Inspect(raw []byte) (Artifact, error) { + var artifact Artifact + keySet, err := Parse(raw) + if err != nil { + return artifact, err + } + digest := sha256.Sum256(raw) + artifact.Data = append([]byte(nil), raw...) + artifact.SHA256 = fmt.Sprintf("%x", digest) + artifact.KeyIDs = make([]string, 0, len(keySet.Keys)) + for _, key := range keySet.Keys { + artifact.KeyIDs = append(artifact.KeyIDs, key.KeyID) + } + return artifact, nil +} + +// Parse decodes and validates an RSA signing-key set used by the OpenShift +// service-account issuer. Unknown top-level fields and trailing data fail +// closed, as do private, duplicate, or incorrectly identified keys. +func Parse(raw []byte) (jose.JSONWebKeySet, error) { + var keySet jose.JSONWebKeySet + fields, err := decodeJSONObjectFields(raw, "JWKS") + if err != nil { + return keySet, fmt.Errorf("decode JWKS: %w", err) + } + for field := range fields { + if field != "keys" { + return keySet, fmt.Errorf("decode JWKS: unknown field %q", field) + } + } + encodedKeys, exists := fields["keys"] + if !exists { + return keySet, fmt.Errorf("decode JWKS: missing field %q", "keys") + } + var keys []json.RawMessage + if err := json.Unmarshal(encodedKeys, &keys); err != nil { + return keySet, fmt.Errorf("decode JWKS keys: %w", err) + } + + keySet.Keys = make([]jose.JSONWebKey, 0, len(keys)) + for index, encodedKey := range keys { + if err := validateEncodedKeyFields(index, encodedKey); err != nil { + return keySet, err + } + var key jose.JSONWebKey + if err := json.Unmarshal(encodedKey, &key); err != nil { + return keySet, fmt.Errorf("decode JWKS key %d: %w", index, err) + } + keySet.Keys = append(keySet.Keys, key) + } + + if err := Validate(keySet); err != nil { + return keySet, err + } + return keySet, nil +} + +func validateEncodedKeyFields(index int, encodedKey json.RawMessage) error { + fields, err := decodeJSONObjectFields(encodedKey, fmt.Sprintf("JWKS key %d", index)) + if err != nil { + return fmt.Errorf("decode JWKS key %d fields: %w", index, err) + } + for field := range fields { + switch field { + case "use", "kty", "kid", "alg", "n", "e", "x5c", "x5u", "x5t", "x5t#S256": + default: + return fmt.Errorf("JWKS key %d contains unsupported field %q", index, field) + } + } + return nil +} + +func decodeJSONObjectFields(raw []byte, context string) (map[string]json.RawMessage, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + token, err := decoder.Token() + if err != nil { + return nil, err + } + opening, ok := token.(json.Delim) + if !ok || opening != '{' { + return nil, fmt.Errorf("%s must be a JSON object", context) + } + + fields := make(map[string]json.RawMessage) + for decoder.More() { + fieldToken, err := decoder.Token() + if err != nil { + return nil, err + } + field, ok := fieldToken.(string) + if !ok { + return nil, fmt.Errorf("%s contains a non-string field name", context) + } + if _, exists := fields[field]; exists { + return nil, fmt.Errorf("%s contains duplicate field %q", context, field) + } + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return nil, err + } + fields[field] = value + } + closing, err := decoder.Token() + if err != nil { + return nil, err + } + if delimiter, ok := closing.(json.Delim); !ok || delimiter != '}' { + return nil, fmt.Errorf("%s has an invalid closing delimiter", context) + } + var trailing interface{} + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return nil, fmt.Errorf("unexpected trailing JSON value") + } + return nil, fmt.Errorf("decode trailing data: %w", err) + } + return fields, nil +} + +// NewSigner creates a one-key JWKS from exactly one PEM-encoded PKIX RSA +// public-key block with no headers and only surrounding whitespace. OpenShift +// public signer ConfigMaps use both PUBLIC KEY and RSA PUBLIC KEY labels for +// PKIX-encoded public key bytes. +func NewSigner(publicPEM []byte) (jose.JSONWebKeySet, error) { + var keySet jose.JSONWebKeySet + trimmedLeading := bytes.TrimLeftFunc(publicPEM, unicode.IsSpace) + block, remainder := pem.Decode(trimmedLeading) + if block == nil { + return keySet, fmt.Errorf("decode signer public key PEM") + } + switch block.Type { + case "PUBLIC KEY", "RSA PUBLIC KEY": + default: + return keySet, fmt.Errorf("signer public key PEM block type must be %q or %q", "PUBLIC KEY", "RSA PUBLIC KEY") + } + if len(block.Headers) != 0 { + return keySet, fmt.Errorf("signer public key PEM must not contain headers") + } + consumed := trimmedLeading[:len(trimmedLeading)-len(remainder)] + openingMarker := []byte("-----BEGIN " + block.Type + "-----") + if bytes.LastIndex(consumed, openingMarker) != 0 { + return keySet, fmt.Errorf("signer public key PEM contains leading data") + } + if len(bytes.TrimSpace(remainder)) != 0 { + return keySet, fmt.Errorf("signer public key PEM contains trailing data") + } + + parsed, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return keySet, fmt.Errorf("parse signer public key: %w", err) + } + publicKey, ok := parsed.(*rsa.PublicKey) + if !ok { + return keySet, fmt.Errorf("signer public key must be RSA") + } + + keyID, err := KeyIDFromPublicKey(publicKey) + if err != nil { + return keySet, err + } + keySet.Keys = []jose.JSONWebKey{{ + Key: publicKey, + KeyID: keyID, + Algorithm: string(jose.RS256), + Use: "sig", + }} + return keySet, nil +} + +// Merge preserves current-key order and appends the replacement once. A retry +// with the same replacement is idempotent; a repeated key ID with different +// key material fails closed. +func Merge(current, replacement jose.JSONWebKeySet) (jose.JSONWebKeySet, error) { + var combined jose.JSONWebKeySet + if err := Validate(current); err != nil { + return combined, fmt.Errorf("validate current JWKS: %w", err) + } + if err := Validate(replacement); err != nil { + return combined, fmt.Errorf("validate replacement JWKS: %w", err) + } + if len(replacement.Keys) != 1 { + return combined, fmt.Errorf("replacement JWKS must contain exactly one key, got %d", len(replacement.Keys)) + } + + replacementKey := replacement.Keys[0] + replacementDER, err := x509.MarshalPKIXPublicKey(replacementKey.Key) + if err != nil { + return combined, fmt.Errorf("serialize replacement public key: %w", err) + } + for _, currentKey := range current.Keys { + if currentKey.KeyID != replacementKey.KeyID { + continue + } + currentDER, err := x509.MarshalPKIXPublicKey(currentKey.Key) + if err != nil { + return combined, fmt.Errorf("serialize current public key %q: %w", currentKey.KeyID, err) + } + if !bytes.Equal(currentDER, replacementDER) { + return combined, fmt.Errorf("key ID %q identifies different public keys", replacementKey.KeyID) + } + combined.Keys = append([]jose.JSONWebKey(nil), current.Keys...) + return combined, nil + } + + combined.Keys = make([]jose.JSONWebKey, 0, len(current.Keys)+1) + combined.Keys = append(combined.Keys, current.Keys...) + combined.Keys = append(combined.Keys, replacementKey) + return combined, nil +} + +// Encode validates and deterministically marshals a JWKS with the metadata used +// by rotation checkpoints. +func Encode(keySet jose.JSONWebKeySet) (Artifact, error) { + var artifact Artifact + if err := Validate(keySet); err != nil { + return artifact, err + } + + encoded, err := json.MarshalIndent(keySet, "", " ") + if err != nil { + return artifact, fmt.Errorf("encode JWKS: %w", err) + } + encoded = append(encoded, '\n') + digest := sha256.Sum256(encoded) + + artifact.Data = encoded + artifact.SHA256 = fmt.Sprintf("%x", digest) + artifact.KeyIDs = make([]string, 0, len(keySet.Keys)) + for _, key := range keySet.Keys { + artifact.KeyIDs = append(artifact.KeyIDs, key.KeyID) + } + return artifact, nil +} + +// Validate verifies the semantic constraints of service-account issuer keys. +func Validate(keySet jose.JSONWebKeySet) error { + if len(keySet.Keys) == 0 { + return fmt.Errorf("JWKS must contain at least one key") + } + + seenKeyIDs := make(map[string]struct{}, len(keySet.Keys)) + for index := range keySet.Keys { + key := &keySet.Keys[index] + if !key.Valid() { + return fmt.Errorf("JWKS key %d is invalid", index) + } + if !key.IsPublic() { + return fmt.Errorf("JWKS key %d must contain public key material only", index) + } + publicKey, ok := key.Key.(*rsa.PublicKey) + if !ok { + return fmt.Errorf("JWKS key %d must be RSA", index) + } + if key.KeyID == "" || strings.TrimSpace(key.KeyID) != key.KeyID { + return fmt.Errorf("JWKS key %d must have a non-empty key ID without surrounding whitespace", index) + } + expectedKeyID, err := KeyIDFromPublicKey(publicKey) + if err != nil { + return fmt.Errorf("derive JWKS key %d ID: %w", index, err) + } + if key.KeyID != expectedKeyID { + return fmt.Errorf("JWKS key %d ID %q does not match its public key", index, key.KeyID) + } + if key.Algorithm != "" && key.Algorithm != string(jose.RS256) { + return fmt.Errorf("JWKS key %q uses unsupported algorithm %q", key.KeyID, key.Algorithm) + } + if key.Use != "" && key.Use != "sig" { + return fmt.Errorf("JWKS key %q uses unsupported purpose %q", key.KeyID, key.Use) + } + if _, exists := seenKeyIDs[key.KeyID]; exists { + return fmt.Errorf("JWKS contains duplicate key ID %q", key.KeyID) + } + seenKeyIDs[key.KeyID] = struct{}{} + } + return nil +} + +// KeyIDFromPublicKey derives the Kubernetes/OpenShift service-account key ID +// from the SHA-256 digest of the PKIX DER public key. +func KeyIDFromPublicKey(publicKey interface{}) (string, error) { + publicKeyDER, err := x509.MarshalPKIXPublicKey(publicKey) + if err != nil { + return "", fmt.Errorf("serialize public key to DER: %w", err) + } + digest := sha256.Sum256(publicKeyDER) + return base64.RawURLEncoding.EncodeToString(digest[:]), nil +} diff --git a/pkg/cmd/provisioning/jwks/jwks_test.go b/pkg/cmd/provisioning/jwks/jwks_test.go new file mode 100644 index 0000000000..4acec07c3b --- /dev/null +++ b/pkg/cmd/provisioning/jwks/jwks_test.go @@ -0,0 +1,328 @@ +package jwks + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "strings" + "testing" + + jose "github.com/go-jose/go-jose/v4" +) + +func TestNewSignerAndParse(t *testing.T) { + _, keySet := newTestSigner(t) + artifact, err := Encode(keySet) + if err != nil { + t.Fatalf("Encode() returned unexpected error: %v", err) + } + + parsed, err := Parse(artifact.Data) + if err != nil { + t.Fatalf("Parse() returned unexpected error: %v", err) + } + if len(parsed.Keys) != 1 || parsed.Keys[0].KeyID != keySet.Keys[0].KeyID { + t.Fatalf("parsed key set = %#v, want key ID %q", parsed, keySet.Keys[0].KeyID) + } + if parsed.Keys[0].Algorithm != string(jose.RS256) || parsed.Keys[0].Use != "sig" { + t.Fatalf("parsed signer metadata = alg %q use %q", parsed.Keys[0].Algorithm, parsed.Keys[0].Use) + } +} + +func TestNewSignerAcceptsRSAPublicKeyLabelWithPKIXDER(t *testing.T) { + publicPEM := newTestPublicPEM(t) + block, remainder := pem.Decode(publicPEM) + if block == nil || len(remainder) != 0 { + t.Fatal("decode PKIX public-key fixture") + } + standard, err := NewSigner(publicPEM) + if err != nil { + t.Fatalf("NewSigner(PUBLIC KEY) returned unexpected error: %v", err) + } + + rsaLabelPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: block.Bytes}) + got, err := NewSigner(rsaLabelPEM) + if err != nil { + t.Fatalf("NewSigner(RSA PUBLIC KEY with PKIX DER) returned unexpected error: %v", err) + } + if len(got.Keys) != 1 || got.Keys[0].KeyID != standard.Keys[0].KeyID { + t.Fatalf("NewSigner(RSA PUBLIC KEY) = %#v, want key ID %q", got, standard.Keys[0].KeyID) + } +} + +func TestNewSignerRejectsInvalidPublicKey(t *testing.T) { + validPEM := newTestPublicPEM(t) + validBlock, remainder := pem.Decode(validPEM) + if validBlock == nil || len(remainder) != 0 { + t.Fatal("decode valid public-key fixture") + } + parsedPublicKey, err := x509.ParsePKIXPublicKey(validBlock.Bytes) + if err != nil { + t.Fatalf("parse valid public-key fixture: %v", err) + } + rsaPublicKey, ok := parsedPublicKey.(*rsa.PublicKey) + if !ok { + t.Fatalf("valid public-key fixture has type %T, want RSA", parsedPublicKey) + } + wrongType := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: validBlock.Bytes}) + pkcs1Public := pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: x509.MarshalPKCS1PublicKey(rsaPublicKey)}) + privateKey, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + t.Fatalf("generate private-key fixture: %v", err) + } + privateDER, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + t.Fatalf("marshal private-key fixture: %v", err) + } + privateAsPublic := pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: privateDER}) + withHeaders := pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Headers: map[string]string{"Comment": "unexpected metadata"}, + Bytes: validBlock.Bytes, + }) + + tests := []struct { + name string + publicPEM []byte + wantError string + }{ + {name: "not PEM", publicPEM: []byte("not PEM"), wantError: "decode signer public key PEM"}, + {name: "leading junk", publicPEM: append([]byte("untrusted prefix\n"), validPEM...), wantError: "leading data"}, + {name: "wrong block type", publicPEM: wrongType, wantError: `block type must be "PUBLIC KEY" or "RSA PUBLIC KEY"`}, + {name: "PKCS1 public DER", publicPEM: pkcs1Public, wantError: "parse signer public key"}, + {name: "private DER with public label", publicPEM: privateAsPublic, wantError: "parse signer public key"}, + {name: "PEM headers", publicPEM: withHeaders, wantError: "must not contain headers"}, + {name: "trailing data", publicPEM: append(append([]byte(nil), validPEM...), []byte("trailing")...), wantError: "trailing data"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := NewSigner(test.publicPEM) + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("NewSigner() error = %v, want error containing %q", err, test.wantError) + } + }) + } + + ecdsaKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate ECDSA key: %v", err) + } + der, err := x509.MarshalPKIXPublicKey(&ecdsaKey.PublicKey) + if err != nil { + t.Fatalf("marshal ECDSA public key: %v", err) + } + _, err = NewSigner(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})) + if err == nil || !strings.Contains(err.Error(), "must be RSA") { + t.Fatalf("NewSigner(ECDSA) error = %v, want RSA error", err) + } +} + +func TestNewSignerAllowsSurroundingWhitespace(t *testing.T) { + validPEM := newTestPublicPEM(t) + want, err := NewSigner(validPEM) + if err != nil { + t.Fatalf("NewSigner(valid fixture) returned unexpected error: %v", err) + } + wrapped := append([]byte(" \t\r\n"), validPEM...) + wrapped = append(wrapped, []byte("\n\r\t ")...) + + got, err := NewSigner(wrapped) + if err != nil { + t.Fatalf("NewSigner(whitespace-wrapped fixture) returned unexpected error: %v", err) + } + if len(got.Keys) != 1 || got.Keys[0].KeyID != want.Keys[0].KeyID { + t.Fatalf("NewSigner(whitespace-wrapped fixture) key set = %#v, want key ID %q", got, want.Keys[0].KeyID) + } +} + +func TestParseRejectsUnsafeJWKS(t *testing.T) { + privateKey, validSet := newTestSigner(t) + validArtifact, err := Encode(validSet) + if err != nil { + t.Fatalf("encode valid JWKS: %v", err) + } + + wrongKeyID := validSet + wrongKeyID.Keys = append([]jose.JSONWebKey(nil), validSet.Keys...) + wrongKeyID.Keys[0].KeyID = "wrong-key-id" + + unsupportedAlgorithm := validSet + unsupportedAlgorithm.Keys = append([]jose.JSONWebKey(nil), validSet.Keys...) + unsupportedAlgorithm.Keys[0].Algorithm = "RS512" + + unsupportedUse := validSet + unsupportedUse.Keys = append([]jose.JSONWebKey(nil), validSet.Keys...) + unsupportedUse.Keys[0].Use = "enc" + + duplicate := validSet + duplicate.Keys = append(append([]jose.JSONWebKey(nil), validSet.Keys...), validSet.Keys[0]) + + privateSet := jose.JSONWebKeySet{Keys: []jose.JSONWebKey{{ + Key: privateKey, + KeyID: validSet.Keys[0].KeyID, + Algorithm: string(jose.RS256), + Use: "sig", + }}} + + ecdsaKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate ECDSA key: %v", err) + } + ecdsaKeyID, err := KeyIDFromPublicKey(&ecdsaKey.PublicKey) + if err != nil { + t.Fatalf("derive ECDSA key ID: %v", err) + } + ecdsaSet := jose.JSONWebKeySet{Keys: []jose.JSONWebKey{{Key: &ecdsaKey.PublicKey, KeyID: ecdsaKeyID, Use: "sig"}}} + + unknownTopLevel := append([]byte(nil), validArtifact.Data...) + unknownTopLevel = []byte(strings.Replace(string(unknownTopLevel), "{", `{"unexpected":true,`, 1)) + unknownKeyField := []byte(strings.Replace(string(validArtifact.Data), `"kty"`, `"key_ops":["verify"],"kty"`, 1)) + nonRSAKeyField := []byte(strings.Replace(string(validArtifact.Data), `"kty"`, `"crv":"P-256","x":"ignored","kty"`, 1)) + privateFragment := []byte(strings.Replace(string(validArtifact.Data), `"kty"`, `"p":"AQAB","kty"`, 1)) + duplicateTopLevel := []byte(strings.Replace(string(validArtifact.Data), `"keys":`, `"keys":[],"keys":`, 1)) + duplicateKeyID := []byte(strings.Replace(string(validArtifact.Data), `"kid":`, `"kid":"duplicate","kid":`, 1)) + duplicateModulus := []byte(strings.Replace(string(validArtifact.Data), `"n":`, `"n":"AQ","n":`, 1)) + + tests := []struct { + name string + raw []byte + wantError string + }{ + {name: "empty set", raw: []byte(`{"keys":[]}`), wantError: "at least one key"}, + {name: "malformed", raw: []byte(`{"keys":`), wantError: "decode JWKS"}, + {name: "unknown top-level field", raw: unknownTopLevel, wantError: "unknown field"}, + {name: "unsupported key field", raw: unknownKeyField, wantError: "unsupported field"}, + {name: "non-RSA key fields", raw: nonRSAKeyField, wantError: "unsupported field"}, + {name: "private key fragment", raw: privateFragment, wantError: "unsupported field"}, + {name: "duplicate top-level field", raw: duplicateTopLevel, wantError: "duplicate field"}, + {name: "duplicate key ID field", raw: duplicateKeyID, wantError: "duplicate field"}, + {name: "duplicate modulus field", raw: duplicateModulus, wantError: "duplicate field"}, + {name: "trailing JSON", raw: append(validArtifact.Data, []byte(`{}`)...), wantError: "trailing JSON value"}, + {name: "incorrect key ID", raw: mustMarshalKeySet(t, wrongKeyID), wantError: "does not match its public key"}, + {name: "unsupported algorithm", raw: mustMarshalKeySet(t, unsupportedAlgorithm), wantError: "unsupported algorithm"}, + {name: "unsupported use", raw: mustMarshalKeySet(t, unsupportedUse), wantError: "unsupported purpose"}, + {name: "duplicate key ID", raw: mustMarshalKeySet(t, duplicate), wantError: "duplicate key ID"}, + {name: "private key", raw: mustMarshalKeySet(t, privateSet), wantError: "unsupported field"}, + {name: "non-RSA key", raw: mustMarshalKeySet(t, ecdsaSet), wantError: "unsupported field"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := Parse(test.raw) + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("Parse() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestKeyIDFromPublicKeyCompatibilityVector(t *testing.T) { + publicKey := &rsa.PublicKey{N: big.NewInt(3233), E: 17} + keyID, err := KeyIDFromPublicKey(publicKey) + if err != nil { + t.Fatalf("KeyIDFromPublicKey() returned unexpected error: %v", err) + } + const expectedKeyID = "6WtxEDjtjJdrMQPQXDEWDrhLO-JtqWqYWjji-yH4_2w" + if keyID != expectedKeyID { + t.Fatalf("KeyIDFromPublicKey() = %q, want %q", keyID, expectedKeyID) + } +} + +func TestMergeIsOrderedAndIdempotent(t *testing.T) { + _, current := newTestSigner(t) + _, replacement := newTestSigner(t) + + combined, err := Merge(current, replacement) + if err != nil { + t.Fatalf("Merge() returned unexpected error: %v", err) + } + if len(combined.Keys) != 2 { + t.Fatalf("combined key count = %d, want 2", len(combined.Keys)) + } + if combined.Keys[0].KeyID != current.Keys[0].KeyID || combined.Keys[1].KeyID != replacement.Keys[0].KeyID { + t.Fatalf("combined key order = [%q %q], want [%q %q]", combined.Keys[0].KeyID, combined.Keys[1].KeyID, current.Keys[0].KeyID, replacement.Keys[0].KeyID) + } + + retried, err := Merge(combined, replacement) + if err != nil { + t.Fatalf("Merge(retry) returned unexpected error: %v", err) + } + if len(retried.Keys) != 2 { + t.Fatalf("retried key count = %d, want 2", len(retried.Keys)) + } +} + +func TestMergeRejectsInvalidReplacement(t *testing.T) { + _, current := newTestSigner(t) + _, first := newTestSigner(t) + _, second := newTestSigner(t) + replacement := jose.JSONWebKeySet{Keys: []jose.JSONWebKey{first.Keys[0], second.Keys[0]}} + + _, err := Merge(current, replacement) + if err == nil || !strings.Contains(err.Error(), "exactly one key") { + t.Fatalf("Merge() error = %v, want exactly-one-key error", err) + } +} + +func TestEncodeReturnsDigestAndKeyIDs(t *testing.T) { + _, first := newTestSigner(t) + _, second := newTestSigner(t) + keySet := jose.JSONWebKeySet{Keys: []jose.JSONWebKey{first.Keys[0], second.Keys[0]}} + + artifact, err := Encode(keySet) + if err != nil { + t.Fatalf("Encode() returned unexpected error: %v", err) + } + digest := sha256.Sum256(artifact.Data) + if artifact.SHA256 != fmt.Sprintf("%x", digest) { + t.Fatalf("artifact digest = %q, want %x", artifact.SHA256, digest) + } + if len(artifact.KeyIDs) != 2 || artifact.KeyIDs[0] != first.Keys[0].KeyID || artifact.KeyIDs[1] != second.Keys[0].KeyID { + t.Fatalf("artifact key IDs = %v", artifact.KeyIDs) + } +} + +func newTestSigner(t *testing.T) (*rsa.PrivateKey, jose.JSONWebKeySet) { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + t.Fatalf("generate RSA key: %v", err) + } + der, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey) + if err != nil { + t.Fatalf("marshal RSA public key: %v", err) + } + publicPEM := pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der}) + keySet, err := NewSigner(publicPEM) + if err != nil { + t.Fatalf("NewSigner() returned unexpected error: %v", err) + } + return privateKey, keySet +} + +func newTestPublicPEM(t *testing.T) []byte { + t.Helper() + _, keySet := newTestSigner(t) + der, err := x509.MarshalPKIXPublicKey(keySet.Keys[0].Key) + if err != nil { + t.Fatalf("marshal test public key: %v", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der}) +} + +func mustMarshalKeySet(t *testing.T, keySet jose.JSONWebKeySet) []byte { + t.Helper() + raw, err := json.Marshal(keySet) + if err != nil { + t.Fatalf("marshal JWKS fixture: %v", err) + } + return raw +} diff --git a/pkg/cmd/provisioning/rotation/artifact_store.go b/pkg/cmd/provisioning/rotation/artifact_store.go new file mode 100644 index 0000000000..4fe3085d48 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/artifact_store.go @@ -0,0 +1,255 @@ +package rotation + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" +) + +const artifactFileMode = 0o644 + +// StoredArtifact is validated public evidence recorded by a rotation +// checkpoint. Data and metadata are copied before they are returned. +type StoredArtifact struct { + Metadata ArtifactMetadata + Path string + Data []byte +} + +// WriteArtifact validates and durably writes one public rotation artifact. +// Artifact names are immutable within a workspace: an identical retry is +// accepted, while an attempt to replace prior evidence fails closed. +func WriteArtifact(outputDir, name string, payload []byte) (ArtifactMetadata, error) { + var metadata ArtifactMetadata + err := WithRotationWorkspace(outputDir, func(workspace *RotationWorkspace) error { + var err error + metadata, err = workspace.WriteArtifact(name, payload) + return err + }) + if err != nil { + return ArtifactMetadata{}, err + } + return metadata, nil +} + +// WriteArtifact validates and persists a public artifact while retaining the +// workspace lease held by the surrounding orchestration operation. +func (w *RotationWorkspace) WriteArtifact(name string, payload []byte) (ArtifactMetadata, error) { + if err := w.validateActive(); err != nil { + return ArtifactMetadata{}, err + } + payload = append([]byte(nil), payload...) + metadata, err := inspectArtifactPayload(name, payload) + if err != nil { + return ArtifactMetadata{}, err + } + checkpoint, err := w.LoadCheckpoint() + if errors.Is(err, errCheckpointNotFound) { + checkpoint = Checkpoint{OutputDir: w.outputDir} + } else if err != nil { + return ArtifactMetadata{}, fmt.Errorf("load rotation checkpoint before writing artifact: %w", err) + } + + artifactPath := filepath.Join(w.outputDir, name) + existing, existingErr := readExistingArtifact(artifactPath, name) + switch { + case existingErr == nil: + if !bytes.Equal(existing, payload) { + return ArtifactMetadata{}, fmt.Errorf("rotation artifact %q already exists with different content", name) + } + case !errors.Is(existingErr, os.ErrNotExist): + return ArtifactMetadata{}, existingErr + } + + if artifactIsRecorded(checkpoint, name) { + return metadata, nil + } + payloads, err := loadArtifactPrerequisites(w.outputDir, name) + if err != nil { + return ArtifactMetadata{}, err + } + payloads[name] = payload + if err := validateArtifactRelationships(checkpoint, payloads); err != nil { + return ArtifactMetadata{}, err + } + if existingErr == nil && bytes.Equal(existing, payload) { + return metadata, nil + } + + if err := writeFileAtomically(w.outputDir, name, payload, artifactFileMode); err != nil { + return ArtifactMetadata{}, fmt.Errorf("persist rotation artifact %q: %w", name, err) + } + return metadata, nil +} + +// ReadArtifact returns an artifact only when the active checkpoint records the +// same digest and key identities. Unrecorded files are never treated as +// completed phase evidence. +func (w *RotationWorkspace) ReadArtifact(name string) (StoredArtifact, error) { + if err := w.validateActive(); err != nil { + return StoredArtifact{}, err + } + checkpoint, err := w.LoadCheckpoint() + if err != nil { + return StoredArtifact{}, fmt.Errorf("load rotation checkpoint before reading artifact: %w", err) + } + + var recorded ArtifactMetadata + found := false + for _, artifact := range checkpoint.Artifacts { + if artifact.Name == name { + recorded = artifact + found = true + break + } + } + if !found { + return StoredArtifact{}, fmt.Errorf("rotation artifact %q is not recorded by checkpoint phase %q", name, checkpoint.Phase) + } + + artifactPath := filepath.Join(w.outputDir, name) + payload, err := readExistingArtifact(artifactPath, name) + if err != nil { + return StoredArtifact{}, err + } + observed, err := inspectArtifactPayload(name, payload) + if err != nil { + return StoredArtifact{}, err + } + if observed.SHA256 != recorded.SHA256 || !slices.Equal(observed.KeyIDs, recorded.KeyIDs) { + return StoredArtifact{}, fmt.Errorf("rotation artifact %q does not match its recorded metadata", name) + } + + recorded.KeyIDs = append([]string(nil), recorded.KeyIDs...) + return StoredArtifact{ + Metadata: recorded, + Path: artifactPath, + Data: append([]byte(nil), payload...), + }, nil +} + +func artifactIsRecorded(checkpoint Checkpoint, name string) bool { + for _, artifact := range checkpoint.Artifacts { + if artifact.Name == name { + return true + } + } + return false +} + +func loadArtifactPrerequisites(outputDir, name string) (map[string][]byte, error) { + var required []string + switch name { + case ArtifactCurrentJWKS: + case ArtifactReplacementPublicKey: + required = []string{ArtifactCurrentJWKS} + case ArtifactNewJWKS: + required = []string{ArtifactCurrentJWKS, ArtifactReplacementPublicKey} + case ArtifactCombinedJWKS: + required = []string{ArtifactCurrentJWKS, ArtifactReplacementPublicKey, ArtifactNewJWKS} + default: + return nil, fmt.Errorf("unsupported rotation artifact %q", name) + } + + payloads := make(map[string][]byte, len(required)+1) + for _, prerequisite := range required { + payload, err := readExistingArtifact(filepath.Join(outputDir, prerequisite), prerequisite) + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("rotation artifact %q requires existing artifact %q", name, prerequisite) + } + if err != nil { + return nil, err + } + payloads[prerequisite] = payload + } + return payloads, nil +} + +func inspectArtifactPayload(name string, payload []byte) (ArtifactMetadata, error) { + var keyIDs []string + if len(payload) > maxArtifactSize { + return ArtifactMetadata{}, fmt.Errorf("rotation artifact %q exceeds %d bytes", name, maxArtifactSize) + } + + switch name { + case ArtifactReplacementPublicKey: + keySet, err := jwkutil.NewSigner(payload) + if err != nil { + return ArtifactMetadata{}, fmt.Errorf("validate rotation artifact %q: %w", name, err) + } + keyIDs = []string{keySet.Keys[0].KeyID} + case ArtifactCurrentJWKS: + artifact, err := jwkutil.Inspect(payload) + if err != nil { + return ArtifactMetadata{}, fmt.Errorf("validate rotation artifact %q: %w", name, err) + } + keyIDs = artifact.KeyIDs + case ArtifactNewJWKS, ArtifactCombinedJWKS: + keySet, err := jwkutil.Parse(payload) + if err != nil { + return ArtifactMetadata{}, fmt.Errorf("validate rotation artifact %q: %w", name, err) + } + canonical, err := jwkutil.Encode(keySet) + if err != nil { + return ArtifactMetadata{}, fmt.Errorf("encode rotation artifact %q: %w", name, err) + } + if !bytes.Equal(payload, canonical.Data) { + return ArtifactMetadata{}, fmt.Errorf("rotation artifact %q must use canonical encoding", name) + } + keyIDs = canonical.KeyIDs + default: + return ArtifactMetadata{}, fmt.Errorf("unsupported rotation artifact %q", name) + } + + digest := sha256.Sum256(payload) + metadata := ArtifactMetadata{ + Name: name, + SHA256: fmt.Sprintf("%x", digest), + KeyIDs: keyIDs, + } + if err := validateArtifact(metadata); err != nil { + return ArtifactMetadata{}, err + } + return metadata, nil +} + +func readExistingArtifact(path, name string) ([]byte, error) { + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("rotation artifact %q must be a regular file", name) + } + if info.Mode().Perm()&0o022 != 0 { + return nil, fmt.Errorf("rotation artifact %q must not be writable by group or others", name) + } + + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open rotation artifact %q: %w", name, err) + } + defer file.Close() + openedInfo, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("inspect opened rotation artifact %q: %w", name, err) + } + if !os.SameFile(info, openedInfo) { + return nil, fmt.Errorf("rotation artifact %q changed while it was opened", name) + } + payload, err := io.ReadAll(io.LimitReader(file, maxArtifactSize+1)) + if err != nil { + return nil, fmt.Errorf("read rotation artifact %q: %w", name, err) + } + if len(payload) > maxArtifactSize { + return nil, fmt.Errorf("rotation artifact %q exceeds %d bytes", name, maxArtifactSize) + } + return payload, nil +} diff --git a/pkg/cmd/provisioning/rotation/artifact_store_test.go b/pkg/cmd/provisioning/rotation/artifact_store_test.go new file mode 100644 index 0000000000..80632f5c74 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/artifact_store_test.go @@ -0,0 +1,438 @@ +package rotation + +import ( + "encoding/pem" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" +) + +func TestWriteArtifactIsDurableImmutableAndIdempotent(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + currentRaw := encodedJWKSForTest(t, testPublicKeyPEM(t)) + want, err := jwkutil.Inspect(currentRaw) + if err != nil { + t.Fatalf("inspect current JWKS fixture: %v", err) + } + + metadata, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, currentRaw) + if err != nil { + t.Fatalf("WriteArtifact() returned unexpected error: %v", err) + } + if metadata.Name != ArtifactCurrentJWKS || metadata.SHA256 != want.SHA256 || !reflect.DeepEqual(metadata.KeyIDs, want.KeyIDs) { + t.Fatalf("WriteArtifact() metadata = %#v, want digest %q and key IDs %v", metadata, want.SHA256, want.KeyIDs) + } + + artifactPath := filepath.Join(outputDir, ArtifactCurrentJWKS) + written, err := os.ReadFile(artifactPath) + if err != nil { + t.Fatalf("read written artifact: %v", err) + } + if string(written) != string(currentRaw) { + t.Fatal("written artifact did not preserve its exact bytes") + } + info, err := os.Stat(artifactPath) + if err != nil { + t.Fatalf("stat written artifact: %v", err) + } + if got := info.Mode().Perm(); got != artifactFileMode { + t.Fatalf("artifact mode = %o, want %o", got, artifactFileMode) + } + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + checkpoint, err = LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load initialized checkpoint: %v", err) + } + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaselineForKeyID(metadata.KeyIDs[0]) + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save preflight checkpoint: %v", err) + } + checkpoint.Phase = PhaseGuardAcquired + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save guard-acquired checkpoint: %v", err) + } + checkpoint.Phase = PhaseCurrentJWKSRead + checkpoint.Artifacts = []ArtifactMetadata{metadata} + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("record current JWKS: %v", err) + } + + if _, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, currentRaw); err != nil { + t.Fatalf("idempotent WriteArtifact() returned unexpected error: %v", err) + } + different := encodedJWKSForTest(t, testPublicKeyPEM(t)) + if _, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, different); err == nil || !strings.Contains(err.Error(), "already exists with different content") { + t.Fatalf("WriteArtifact(different content) error = %v", err) + } +} + +func TestWriteArtifactRejectsReplacementOfUnrecordedEvidence(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + currentRaw := encodedJWKSForTest(t, testPublicKeyPEM(t)) + currentMetadata, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, currentRaw) + if err != nil { + t.Fatalf("write current JWKS: %v", err) + } + + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + checkpoint, err = LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load initialized checkpoint: %v", err) + } + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaselineForKeyID(currentMetadata.KeyIDs[0]) + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save preflight checkpoint: %v", err) + } + checkpoint.Phase = PhaseGuardAcquired + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save guard-acquired checkpoint: %v", err) + } + checkpoint.Phase = PhaseCurrentJWKSRead + checkpoint.Artifacts = []ArtifactMetadata{currentMetadata} + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save current-JWKS checkpoint: %v", err) + } + checkpoint.Phase = PhaseNextKeyRequested + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save replacement-requested checkpoint: %v", err) + } + + wrongReplacement := testPublicKeyPEM(t) + if _, err := WriteArtifact(outputDir, ArtifactReplacementPublicKey, wrongReplacement); err != nil { + t.Fatalf("write unrecorded replacement fixture: %v", err) + } + correctReplacement := testPublicKeyPEM(t) + if _, err := WriteArtifact(outputDir, ArtifactReplacementPublicKey, correctReplacement); err == nil || !strings.Contains(err.Error(), "already exists with different content") { + t.Fatalf("replace unrecorded replacement error = %v", err) + } + written, err := os.ReadFile(filepath.Join(outputDir, ArtifactReplacementPublicKey)) + if err != nil { + t.Fatalf("read preserved replacement: %v", err) + } + if string(written) != string(wrongReplacement) { + t.Fatal("unrecorded replacement artifact was changed") + } +} + +func TestRotationWorkspaceReadArtifactRequiresRecordedEvidence(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + currentRaw := encodedJWKSForTest(t, testPublicKeyPEM(t)) + currentMetadata, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, currentRaw) + if err != nil { + t.Fatalf("write current JWKS: %v", err) + } + + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + checkpoint, err = LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load initialized checkpoint: %v", err) + } + + err = WithRotationWorkspace(outputDir, func(workspace *RotationWorkspace) error { + if _, err := workspace.ReadArtifact(ArtifactCurrentJWKS); err == nil || !strings.Contains(err.Error(), "is not recorded") { + t.Fatalf("ReadArtifact(unrecorded) error = %v", err) + } + return nil + }) + if err != nil { + t.Fatalf("inspect unrecorded artifact: %v", err) + } + + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaselineForKeyID(currentMetadata.KeyIDs[0]) + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save preflight checkpoint: %v", err) + } + checkpoint.Phase = PhaseGuardAcquired + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save guard-acquired checkpoint: %v", err) + } + checkpoint.Phase = PhaseCurrentJWKSRead + checkpoint.Artifacts = []ArtifactMetadata{currentMetadata} + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("record current JWKS: %v", err) + } + + var stored StoredArtifact + err = WithRotationWorkspace(outputDir, func(workspace *RotationWorkspace) error { + var err error + stored, err = workspace.ReadArtifact(ArtifactCurrentJWKS) + return err + }) + if err != nil { + t.Fatalf("ReadArtifact(recorded) returned unexpected error: %v", err) + } + if stored.Path != filepath.Join(outputDir, ArtifactCurrentJWKS) || stored.Metadata.SHA256 != currentMetadata.SHA256 || !reflect.DeepEqual(stored.Metadata.KeyIDs, currentMetadata.KeyIDs) || string(stored.Data) != string(currentRaw) { + t.Fatalf("ReadArtifact(recorded) = %#v", stored) + } + + stored.Data[0] ^= 0xff + stored.Metadata.KeyIDs[0] = "changed" + written, err := os.ReadFile(filepath.Join(outputDir, ArtifactCurrentJWKS)) + if err != nil { + t.Fatalf("read current JWKS after mutating result: %v", err) + } + if string(written) != string(currentRaw) || checkpoint.Artifacts[0].KeyIDs[0] == "changed" { + t.Fatal("ReadArtifact returned aliases to persisted evidence") + } +} + +func TestWriteArtifactRejectsInvalidInput(t *testing.T) { + validSingle := encodedJWKSForTest(t, testPublicKeyPEM(t)) + tests := []struct { + name string + artifactName string + payload []byte + wantError string + }{ + {name: "unsupported name", artifactName: "private.key", payload: validSingle, wantError: "unsupported rotation artifact"}, + {name: "invalid public key", artifactName: ArtifactReplacementPublicKey, payload: []byte("not PEM"), wantError: "decode signer public key PEM"}, + {name: "invalid current JWKS", artifactName: ArtifactCurrentJWKS, payload: []byte(`{"keys":[]}`), wantError: "at least one key"}, + {name: "noncanonical new JWKS", artifactName: ArtifactNewJWKS, payload: append([]byte(" \n"), validSingle...), wantError: "canonical encoding"}, + {name: "single-key combined JWKS", artifactName: ArtifactCombinedJWKS, payload: validSingle, wantError: "at least two key IDs"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := WriteArtifact(filepath.Join(t.TempDir(), "rotation-output"), test.artifactName, test.payload) + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("WriteArtifact() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestWriteArtifactDoesNotPersistUnvalidatedPublicKeyEnvelope(t *testing.T) { + validPEM := testPublicKeyPEM(t) + block, remainder := pem.Decode(validPEM) + if block == nil || len(remainder) != 0 { + t.Fatal("decode valid replacement-public-key fixture") + } + withHeaders := pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Headers: map[string]string{"Comment": "unexpected metadata"}, + Bytes: block.Bytes, + }) + tests := []struct { + name string + payload []byte + }{ + {name: "leading data", payload: append([]byte("untrusted prefix\n"), validPEM...)}, + {name: "PEM headers", payload: withHeaders}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + _, err := WriteArtifact(outputDir, ArtifactReplacementPublicKey, test.payload) + if err == nil { + t.Fatal("WriteArtifact() unexpectedly accepted an invalid public-key envelope") + } + artifactPath := filepath.Join(outputDir, ArtifactReplacementPublicKey) + if _, statErr := os.Lstat(artifactPath); !os.IsNotExist(statErr) { + t.Fatalf("invalid public-key envelope persisted at %q: %v", artifactPath, statErr) + } + }) + } +} + +func TestWriteArtifactRejectsUnsafeExistingPath(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + if err := ensureCheckpointDirectory(outputDir); err != nil { + t.Fatalf("create checkpoint directory: %v", err) + } + target := filepath.Join(outputDir, "target.json") + if err := os.WriteFile(target, []byte("target"), artifactFileMode); err != nil { + t.Fatalf("write symlink target: %v", err) + } + if err := os.Symlink(target, filepath.Join(outputDir, ArtifactCurrentJWKS)); err != nil { + t.Fatalf("create artifact symlink: %v", err) + } + + validCurrent := encodedJWKSForTest(t, testPublicKeyPEM(t)) + _, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, validCurrent) + if err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("WriteArtifact(symlink) error = %v", err) + } +} + +func TestWriteArtifactRejectsWritableExistingArtifact(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + currentRaw := encodedJWKSForTest(t, testPublicKeyPEM(t)) + if _, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, currentRaw); err != nil { + t.Fatalf("write current JWKS: %v", err) + } + artifactPath := filepath.Join(outputDir, ArtifactCurrentJWKS) + if err := os.Chmod(artifactPath, 0o664); err != nil { + t.Fatalf("make current JWKS group-writable: %v", err) + } + + _, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, currentRaw) + if err == nil || !strings.Contains(err.Error(), "must not be writable") { + t.Fatalf("WriteArtifact(writable artifact) error = %v", err) + } +} + +func TestCheckpointAndArtifactStoresRejectWritableWorkspace(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + if err := os.Mkdir(outputDir, 0o700); err != nil { + t.Fatalf("create output directory: %v", err) + } + if err := os.Chmod(outputDir, 0o770); err != nil { + t.Fatalf("make output directory group-writable: %v", err) + } + + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err == nil || !strings.Contains(err.Error(), "must not be writable") { + t.Fatalf("SaveCheckpoint(writable workspace) error = %v", err) + } + validCurrent := encodedJWKSForTest(t, testPublicKeyPEM(t)) + if _, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, validCurrent); err == nil || !strings.Contains(err.Error(), "must not be writable") { + t.Fatalf("WriteArtifact(writable workspace) error = %v", err) + } +} + +func TestCheckpointStorePersistsValidatedArtifactBundle(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + + currentRaw := encodedJWKSForTest(t, testPublicKeyPEM(t)) + replacementPEM := testPublicKeyPEM(t) + prepared, err := PrepareJWKSArtifacts(currentRaw, replacementPEM) + if err != nil { + t.Fatalf("prepare JWKS artifact bundle: %v", err) + } + + checkpoint, err = LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load initialized checkpoint: %v", err) + } + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaselineForKeyID(prepared.Current.KeyIDs[0]) + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save preflight checkpoint: %v", err) + } + checkpoint.Phase = PhaseGuardAcquired + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save guard-acquired checkpoint: %v", err) + } + + currentMetadata, err := WriteArtifact(outputDir, ArtifactCurrentJWKS, prepared.Current.Data) + if err != nil { + t.Fatalf("write current JWKS: %v", err) + } + checkpoint.Phase = PhaseCurrentJWKSRead + checkpoint.Artifacts = append(checkpoint.Artifacts, currentMetadata) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save current-JWKS checkpoint: %v", err) + } + + checkpoint.Phase = PhaseNextKeyRequested + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save replacement-requested checkpoint: %v", err) + } + + replacementMetadata, err := WriteArtifact(outputDir, ArtifactReplacementPublicKey, replacementPEM) + if err != nil { + t.Fatalf("write replacement public key: %v", err) + } + checkpoint.Phase = PhaseNextPublicKeyRead + checkpoint.Artifacts = append(checkpoint.Artifacts, replacementMetadata) + checkpoint.ReplacementSigner = testReplacementEvidenceForMetadata(replacementMetadata) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save replacement-public-key checkpoint: %v", err) + } + + newMetadata, err := WriteArtifact(outputDir, ArtifactNewJWKS, prepared.New.Data) + if err != nil { + t.Fatalf("write new JWKS: %v", err) + } + checkpoint.Phase = PhaseNewJWKSBuilt + checkpoint.Artifacts = append(checkpoint.Artifacts, newMetadata) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save new-JWKS checkpoint: %v", err) + } + + combinedMetadata, err := WriteArtifact(outputDir, ArtifactCombinedJWKS, prepared.Combined.Data) + if err != nil { + t.Fatalf("write combined JWKS: %v", err) + } + checkpoint.Phase = PhaseCombinedJWKSBuilt + checkpoint.Artifacts = append(checkpoint.Artifacts, combinedMetadata) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save combined-JWKS checkpoint: %v", err) + } + + loaded, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load validated artifact bundle: %v", err) + } + if loaded.Phase != PhaseCombinedJWKSBuilt || !reflect.DeepEqual(loaded.Artifacts, checkpoint.Artifacts) { + t.Fatalf("loaded checkpoint = %#v, want phase %q and artifacts %#v", loaded, PhaseCombinedJWKSBuilt, checkpoint.Artifacts) + } + + checkpoint.Phase = PhaseCombinedJWKSPublished + checkpoint.Publications = append(checkpoint.Publications, PublicationConfirmation{ + Phase: PhaseCombinedJWKSPublished, + Artifact: ArtifactCombinedJWKS, + SHA256: combinedMetadata.SHA256, + }) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save combined-publication checkpoint: %v", err) + } + checkpoint.Phase = PhaseSignerRolloutStable + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save signer-rollout checkpoint: %v", err) + } + checkpoint.Phase = PhaseRebootIntentRecorded + checkpoint.RebootIntent = testNodeRebootIntent() + checkpoint.RebootIntent.ID, err = rebootIntentID(checkpoint.ClusterIdentity, replacementMetadata.KeyIDs[0]) + if err != nil { + t.Fatalf("derive reboot intent ID: %v", err) + } + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save reboot-intent checkpoint: %v", err) + } + + loaded, err = LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load reboot-intent checkpoint: %v", err) + } + if loaded.Phase != PhaseRebootIntentRecorded || !reflect.DeepEqual(loaded.RebootIntent, checkpoint.RebootIntent) { + t.Fatalf("loaded checkpoint = %#v, want phase %q and reboot intent %#v", loaded, PhaseRebootIntentRecorded, checkpoint.RebootIntent) + } +} diff --git a/pkg/cmd/provisioning/rotation/checkpoint_lock.go b/pkg/cmd/provisioning/rotation/checkpoint_lock.go new file mode 100644 index 0000000000..356dbdff65 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/checkpoint_lock.go @@ -0,0 +1,41 @@ +package rotation + +import ( + "errors" + "fmt" + "os" +) + +const checkpointLockFileName = ".rotation-state.lock" + +// ErrCheckpointLocked indicates that another process is updating the same +// rotation workspace. Callers should fail instead of waiting indefinitely. +var ErrCheckpointLocked = errors.New("rotation checkpoint is locked") + +func withCheckpointLock(outputDir string, operation func() error) (returnErr error) { + lockFile, err := acquireCheckpointLock(outputDir) + if err != nil { + return err + } + defer func() { + if err := releaseCheckpointLock(lockFile); err != nil { + returnErr = errors.Join(returnErr, fmt.Errorf("release rotation checkpoint lock: %w", err)) + } + }() + + return operation() +} + +func validateCheckpointLockFile(file *os.File) error { + info, err := file.Stat() + if err != nil { + return fmt.Errorf("inspect rotation checkpoint lock: %w", err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("rotation checkpoint lock %q must be a regular file", file.Name()) + } + if info.Mode().Perm() != checkpointFileMode { + return fmt.Errorf("rotation checkpoint lock %q permissions must be %o", file.Name(), checkpointFileMode) + } + return nil +} diff --git a/pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go b/pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go new file mode 100644 index 0000000000..1db9bb9ad5 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go @@ -0,0 +1,62 @@ +//go:build linux || darwin + +package rotation + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +func acquireCheckpointLock(outputDir string) (*os.File, error) { + lockPath := outputDir + string(os.PathSeparator) + checkpointLockFileName + flags := unix.O_RDWR | unix.O_CLOEXEC | unix.O_NOFOLLOW + fd, err := unix.Open(lockPath, flags|unix.O_CREAT|unix.O_EXCL, uint32(checkpointFileMode)) + created := err == nil + if errors.Is(err, unix.EEXIST) { + fd, err = unix.Open(lockPath, flags, 0) + } + if err != nil { + return nil, fmt.Errorf("open rotation checkpoint lock %q: %w", lockPath, err) + } + + lockFile := os.NewFile(uintptr(fd), lockPath) + if lockFile == nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("open rotation checkpoint lock %q", lockPath) + } + closeWithError := func(operationErr error) (*os.File, error) { + if closeErr := lockFile.Close(); closeErr != nil { + return nil, errors.Join(operationErr, fmt.Errorf("close rotation checkpoint lock: %w", closeErr)) + } + return nil, operationErr + } + + if created { + if err := lockFile.Chmod(checkpointFileMode); err != nil { + return closeWithError(fmt.Errorf("set rotation checkpoint lock permissions: %w", err)) + } + if err := syncDirectory(outputDir); err != nil { + return closeWithError(err) + } + } + if err := validateCheckpointLockFile(lockFile); err != nil { + return closeWithError(err) + } + if err := unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB); err != nil { + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return closeWithError(fmt.Errorf("%w: %q", ErrCheckpointLocked, lockPath)) + } + return closeWithError(fmt.Errorf("lock rotation checkpoint %q: %w", lockPath, err)) + } + + return lockFile, nil +} + +func releaseCheckpointLock(lockFile *os.File) error { + unlockErr := unix.Flock(int(lockFile.Fd()), unix.LOCK_UN) + closeErr := lockFile.Close() + return errors.Join(unlockErr, closeErr) +} diff --git a/pkg/cmd/provisioning/rotation/checkpoint_lock_unix_test.go b/pkg/cmd/provisioning/rotation/checkpoint_lock_unix_test.go new file mode 100644 index 0000000000..25a03d4184 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/checkpoint_lock_unix_test.go @@ -0,0 +1,87 @@ +//go:build linux || darwin + +package rotation + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestSaveCheckpointRejectsConcurrentWriter(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + if err := ensureCheckpointDirectory(outputDir); err != nil { + t.Fatalf("create checkpoint directory: %v", err) + } + lockFile, err := acquireCheckpointLock(outputDir) + if err != nil { + t.Fatalf("acquire first checkpoint lock: %v", err) + } + + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + err = SaveCheckpoint(checkpoint) + if err == nil || !errors.Is(err, ErrCheckpointLocked) { + _ = releaseCheckpointLock(lockFile) + t.Fatalf("SaveCheckpoint() error = %v, want ErrCheckpointLocked", err) + } + if err := releaseCheckpointLock(lockFile); err != nil { + t.Fatalf("release first checkpoint lock: %v", err) + } + + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("SaveCheckpoint() after release returned unexpected error: %v", err) + } +} + +func TestCheckpointLockRejectsSymlink(t *testing.T) { + outputDir := t.TempDir() + target := filepath.Join(outputDir, "lock-target") + if err := os.WriteFile(target, nil, checkpointFileMode); err != nil { + t.Fatalf("write lock target: %v", err) + } + if err := os.Symlink(target, filepath.Join(outputDir, checkpointLockFileName)); err != nil { + t.Fatalf("create checkpoint lock symlink: %v", err) + } + + _, err := acquireCheckpointLock(outputDir) + if err == nil || !strings.Contains(err.Error(), "open rotation checkpoint lock") { + t.Fatalf("acquireCheckpointLock(symlink) error = %v", err) + } +} + +func TestCheckpointLockRejectsInsecurePermissions(t *testing.T) { + outputDir := t.TempDir() + lockPath := filepath.Join(outputDir, checkpointLockFileName) + if err := os.WriteFile(lockPath, nil, 0o644); err != nil { + t.Fatalf("write checkpoint lock: %v", err) + } + if err := os.Chmod(lockPath, 0o644); err != nil { + t.Fatalf("set insecure checkpoint lock permissions: %v", err) + } + + _, err := acquireCheckpointLock(outputDir) + if err == nil || !strings.Contains(err.Error(), "permissions must be 600") { + t.Fatalf("acquireCheckpointLock(insecure permissions) error = %v", err) + } +} + +func TestCheckpointLockIsReleasedAfterOperationError(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + if err := ensureCheckpointDirectory(outputDir); err != nil { + t.Fatalf("create checkpoint directory: %v", err) + } + wantErr := errors.New("operation failed") + if err := withCheckpointLock(outputDir, func() error { return wantErr }); !errors.Is(err, wantErr) { + t.Fatalf("withCheckpointLock() error = %v, want %v", err, wantErr) + } + + lockFile, err := acquireCheckpointLock(outputDir) + if err != nil { + t.Fatalf("acquire checkpoint lock after operation error: %v", err) + } + if err := releaseCheckpointLock(lockFile); err != nil { + t.Fatalf("release checkpoint lock: %v", err) + } +} diff --git a/pkg/cmd/provisioning/rotation/checkpoint_lock_unsupported.go b/pkg/cmd/provisioning/rotation/checkpoint_lock_unsupported.go new file mode 100644 index 0000000000..b76ff0bd82 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/checkpoint_lock_unsupported.go @@ -0,0 +1,17 @@ +//go:build !linux && !darwin + +package rotation + +import ( + "fmt" + "os" + "runtime" +) + +func acquireCheckpointLock(string) (*os.File, error) { + return nil, fmt.Errorf("rotation checkpoint locking is not supported on %s", runtime.GOOS) +} + +func releaseCheckpointLock(lockFile *os.File) error { + return lockFile.Close() +} diff --git a/pkg/cmd/provisioning/rotation/checkpoint_store.go b/pkg/cmd/provisioning/rotation/checkpoint_store.go new file mode 100644 index 0000000000..4baca9bc05 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/checkpoint_store.go @@ -0,0 +1,541 @@ +package rotation + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" +) + +const ( + CheckpointFileName = "rotation-state.json" + maxCheckpointSize = 1 << 20 + maxArtifactSize = 10 << 20 + checkpointFileMode = 0o600 + checkpointDirMode = 0o700 +) + +var errCheckpointNotFound = errors.New("rotation checkpoint file not found") + +// SaveCheckpoint validates and atomically persists a checkpoint. Existing +// checkpoints may stay at the same phase or advance by exactly one phase; they +// cannot regress, skip a phase, change identities, or rewrite prior evidence. +func SaveCheckpoint(checkpoint Checkpoint) error { + if err := checkpoint.Validate(); err != nil { + return fmt.Errorf("invalid rotation checkpoint: %w", err) + } + return WithRotationWorkspace(checkpoint.OutputDir, func(workspace *RotationWorkspace) error { + return workspace.SaveCheckpoint(checkpoint) + }) +} + +// SaveCheckpoint validates and persists a checkpoint while retaining the +// workspace lease held by the surrounding orchestration operation. +func (w *RotationWorkspace) SaveCheckpoint(checkpoint Checkpoint) error { + return w.saveCheckpoint(checkpoint, false) +} + +// saveCheckpointAdoptingCanonicalRebootIntent is reserved for replacing a +// same-ID local reboot proposal with the cluster-durable canonical intent while +// the checkpoint is still at PhaseRebootIntentRecorded. +func (w *RotationWorkspace) saveCheckpointAdoptingCanonicalRebootIntent(checkpoint Checkpoint) error { + return w.saveCheckpoint(checkpoint, true) +} + +func (w *RotationWorkspace) saveCheckpoint(checkpoint Checkpoint, allowCanonicalRebootIntentAdoption bool) error { + if err := w.validateActive(); err != nil { + return err + } + outputDir, err := resolveRotationOutputDir(checkpoint.OutputDir) + if err != nil { + return err + } + if outputDir != w.outputDir { + return fmt.Errorf("rotation checkpoint output directory %q does not match locked workspace %q", outputDir, w.outputDir) + } + checkpoint.OutputDir = w.outputDir + if err := checkpoint.Validate(); err != nil { + return fmt.Errorf("invalid rotation checkpoint: %w", err) + } + if err := validateArtifactFiles(checkpoint); err != nil { + return err + } + + existing, err := w.LoadCheckpoint() + switch { + case err == nil: + if err := validateCheckpointTransitionWithCanonicalRebootIntentAdoption(existing, checkpoint, allowCanonicalRebootIntentAdoption); err != nil { + return err + } + case !errors.Is(err, errCheckpointNotFound): + return fmt.Errorf("load existing rotation checkpoint: %w", err) + case checkpoint.Phase != PhaseInitialized: + return fmt.Errorf("first rotation checkpoint must use phase %q, got %q", PhaseInitialized, checkpoint.Phase) + } + + payload, err := json.MarshalIndent(checkpoint, "", " ") + if err != nil { + return fmt.Errorf("encode rotation checkpoint: %w", err) + } + payload = append(payload, '\n') + if len(payload) > maxCheckpointSize { + return fmt.Errorf("encoded rotation checkpoint exceeds %d bytes", maxCheckpointSize) + } + + if err := writeFileAtomically(checkpoint.OutputDir, CheckpointFileName, payload, checkpointFileMode); err != nil { + return fmt.Errorf("persist rotation checkpoint: %w", err) + } + + return nil +} + +// LoadCheckpoint reads a checkpoint from outputDir, rejects unsafe file modes +// and unknown JSON fields, binds it to that directory, and validates its state. +func LoadCheckpoint(outputDir string) (Checkpoint, error) { + resolvedOutputDir, err := resolveRotationOutputDir(outputDir) + if err != nil { + return Checkpoint{}, err + } + if err := validateCheckpointDirectory(resolvedOutputDir); err != nil { + return Checkpoint{}, err + } + return loadCheckpoint(resolvedOutputDir) +} + +// LoadCheckpoint reads the state bound to an active workspace lease. +func (w *RotationWorkspace) LoadCheckpoint() (Checkpoint, error) { + if err := w.validateActive(); err != nil { + return Checkpoint{}, err + } + return loadCheckpoint(w.outputDir) +} + +func loadCheckpoint(resolvedOutputDir string) (Checkpoint, error) { + var checkpoint Checkpoint + checkpointPath := filepath.Join(resolvedOutputDir, CheckpointFileName) + + info, err := os.Lstat(checkpointPath) + if errors.Is(err, os.ErrNotExist) { + return checkpoint, fmt.Errorf("%w: %q", errCheckpointNotFound, checkpointPath) + } + if err != nil { + return checkpoint, fmt.Errorf("inspect rotation checkpoint %q: %w", checkpointPath, err) + } + if !info.Mode().IsRegular() { + return checkpoint, fmt.Errorf("rotation checkpoint %q must be a regular file", checkpointPath) + } + if info.Mode().Perm()&0o077 != 0 { + return checkpoint, fmt.Errorf("rotation checkpoint %q permissions must not allow group or other access", checkpointPath) + } + + file, err := os.Open(checkpointPath) + if err != nil { + return checkpoint, fmt.Errorf("open rotation checkpoint: %w", err) + } + defer file.Close() + openedInfo, err := file.Stat() + if err != nil { + return checkpoint, fmt.Errorf("inspect opened rotation checkpoint: %w", err) + } + if !os.SameFile(info, openedInfo) { + return checkpoint, fmt.Errorf("rotation checkpoint %q changed while it was opened", checkpointPath) + } + + payload, err := io.ReadAll(io.LimitReader(file, maxCheckpointSize+1)) + if err != nil { + return checkpoint, fmt.Errorf("read rotation checkpoint: %w", err) + } + if len(payload) > maxCheckpointSize { + return checkpoint, fmt.Errorf("rotation checkpoint exceeds %d bytes", maxCheckpointSize) + } + + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&checkpoint); err != nil { + return checkpoint, fmt.Errorf("decode rotation checkpoint: %w", err) + } + var trailing interface{} + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return checkpoint, fmt.Errorf("decode rotation checkpoint: unexpected trailing JSON value") + } + return checkpoint, fmt.Errorf("decode rotation checkpoint trailing data: %w", err) + } + + checkpointOutputDir, err := filepath.Abs(checkpoint.OutputDir) + if err != nil { + return checkpoint, fmt.Errorf("resolve recorded rotation output directory: %w", err) + } + if filepath.Clean(checkpointOutputDir) != resolvedOutputDir { + return checkpoint, fmt.Errorf("rotation checkpoint output directory %q does not match requested directory %q", checkpoint.OutputDir, resolvedOutputDir) + } + checkpoint.OutputDir = resolvedOutputDir + + if err := checkpoint.Validate(); err != nil { + return checkpoint, fmt.Errorf("invalid rotation checkpoint: %w", err) + } + if err := validateArtifactFiles(checkpoint); err != nil { + return checkpoint, err + } + + return checkpoint, nil +} + +func ensureCheckpointDirectory(outputDir string) error { + info, err := os.Lstat(outputDir) + if err == nil { + return validateCheckpointDirectoryInfo(outputDir, info) + } + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect rotation output directory: %w", err) + } + if err := createCheckpointDirectories(outputDir); err != nil { + return err + } + return validateCheckpointDirectory(outputDir) +} + +func createCheckpointDirectories(outputDir string) error { + missing := []string{outputDir} + parent := filepath.Dir(outputDir) + for { + info, err := os.Stat(parent) + if err == nil { + if !info.IsDir() { + return fmt.Errorf("rotation output directory parent %q is not a directory", parent) + } + break + } + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect rotation output directory parent %q: %w", parent, err) + } + missing = append(missing, parent) + nextParent := filepath.Dir(parent) + if nextParent == parent { + return fmt.Errorf("find existing parent for rotation output directory %q", outputDir) + } + parent = nextParent + } + + for index := len(missing) - 1; index >= 0; index-- { + directory := missing[index] + err := os.Mkdir(directory, checkpointDirMode) + created := err == nil + if err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("create rotation output directory %q: %w", directory, err) + } + if created { + if err := os.Chmod(directory, checkpointDirMode); err != nil { + return fmt.Errorf("set rotation output directory %q permissions: %w", directory, err) + } + } + info, err := os.Lstat(directory) + if err != nil { + return fmt.Errorf("inspect created rotation output directory %q: %w", directory, err) + } + if err := validateCheckpointDirectoryInfo(directory, info); err != nil { + return err + } + if created { + if err := syncDirectory(filepath.Dir(directory)); err != nil { + return fmt.Errorf("persist rotation output directory %q: %w", directory, err) + } + } + } + return nil +} + +func validateCheckpointDirectory(outputDir string) error { + info, err := os.Lstat(outputDir) + if err != nil { + return fmt.Errorf("inspect rotation output directory: %w", err) + } + return validateCheckpointDirectoryInfo(outputDir, info) +} + +func validateCheckpointDirectoryInfo(outputDir string, info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return fmt.Errorf("rotation output directory %q must be a directory and not a symbolic link", outputDir) + } + if info.Mode().Perm()&0o022 != 0 { + return fmt.Errorf("rotation output directory %q must not be writable by group or others", outputDir) + } + return nil +} + +func validateCheckpointTransition(previous, next Checkpoint) error { + return validateCheckpointTransitionWithCanonicalRebootIntentAdoption(previous, next, false) +} + +func validateCheckpointTransitionWithCanonicalRebootIntentAdoption(previous, next Checkpoint, allowCanonicalRebootIntentAdoption bool) error { + if previous.Provider != next.Provider { + return fmt.Errorf("rotation checkpoint provider cannot change from %q to %q", previous.Provider, next.Provider) + } + if previous.PublicationMode != next.PublicationMode { + return fmt.Errorf("rotation checkpoint publication mode cannot change from %q to %q", previous.PublicationMode, next.PublicationMode) + } + if previous.OutputDir != next.OutputDir { + return fmt.Errorf("rotation checkpoint output directory cannot change") + } + if previous.ClusterIdentity != "" && previous.ClusterIdentity != next.ClusterIdentity { + return fmt.Errorf("rotation checkpoint cluster identity cannot change") + } + if previous.TargetIdentity != "" && previous.TargetIdentity != next.TargetIdentity { + return fmt.Errorf("rotation checkpoint target identity cannot change") + } + if previous.PreRotationSignerBaseline != nil && !reflect.DeepEqual(previous.PreRotationSignerBaseline, next.PreRotationSignerBaseline) { + return fmt.Errorf("rotation checkpoint pre-rotation public signer baseline cannot change") + } + if previous.PreRotationSignerRef != nil && !reflect.DeepEqual(previous.PreRotationSignerRef, next.PreRotationSignerRef) { + return fmt.Errorf("rotation checkpoint pre-rotation signer object reference cannot change") + } + if previous.RotationGuard != nil && !reflect.DeepEqual(previous.RotationGuard, next.RotationGuard) { + return fmt.Errorf("rotation checkpoint signer-rotation guard reference cannot change") + } + if previous.ReplacementSigner != nil && !reflect.DeepEqual(previous.ReplacementSigner, next.ReplacementSigner) { + return fmt.Errorf("rotation checkpoint replacement signer evidence cannot change") + } + if previous.RebootIntent != nil && !reflect.DeepEqual(previous.RebootIntent, next.RebootIntent) { + canAdoptCanonicalIntent := allowCanonicalRebootIntentAdoption && + previous.Phase == PhaseRebootIntentRecorded && next.Phase == PhaseRebootIntentRecorded && + next.RebootIntent != nil && previous.RebootIntent.ID == next.RebootIntent.ID + if !canAdoptCanonicalIntent { + return fmt.Errorf("rotation checkpoint reboot intent cannot change") + } + } + + previousPosition := phasePosition(previous.Phase) + nextPosition := phasePosition(next.Phase) + if nextPosition < previousPosition { + return fmt.Errorf("rotation checkpoint cannot regress from phase %q to %q", previous.Phase, next.Phase) + } + if nextPosition > previousPosition+1 { + return fmt.Errorf("rotation checkpoint cannot skip from phase %q to %q", previous.Phase, next.Phase) + } + + if err := validatePreservedArtifacts(previous.Artifacts, next.Artifacts); err != nil { + return err + } + if err := validatePreservedPublications(previous.Publications, next.Publications); err != nil { + return err + } + + return nil +} + +func validatePreservedArtifacts(previous, next []ArtifactMetadata) error { + nextByName := make(map[string]ArtifactMetadata, len(next)) + for _, artifact := range next { + nextByName[artifact.Name] = artifact + } + for _, artifact := range previous { + nextArtifact, exists := nextByName[artifact.Name] + if !exists { + return fmt.Errorf("rotation checkpoint cannot remove recorded artifact %q", artifact.Name) + } + if !reflect.DeepEqual(artifact, nextArtifact) { + return fmt.Errorf("rotation checkpoint cannot change recorded artifact %q", artifact.Name) + } + } + return nil +} + +func validatePreservedPublications(previous, next []PublicationConfirmation) error { + nextByPhase := make(map[Phase]PublicationConfirmation, len(next)) + for _, publication := range next { + nextByPhase[publication.Phase] = publication + } + for _, publication := range previous { + nextPublication, exists := nextByPhase[publication.Phase] + if !exists { + return fmt.Errorf("rotation checkpoint cannot remove publication confirmation for phase %q", publication.Phase) + } + if publication != nextPublication { + return fmt.Errorf("rotation checkpoint cannot change publication confirmation for phase %q", publication.Phase) + } + } + return nil +} + +func validateArtifactFiles(checkpoint Checkpoint) error { + payloads := make(map[string][]byte, len(checkpoint.Artifacts)) + for _, artifact := range checkpoint.Artifacts { + artifactPath := filepath.Join(checkpoint.OutputDir, artifact.Name) + payload, err := readExistingArtifact(artifactPath, artifact.Name) + if err != nil { + return fmt.Errorf("validate recorded rotation artifact %q: %w", artifact.Name, err) + } + + digest := sha256.Sum256(payload) + if fmt.Sprintf("%x", digest) != artifact.SHA256 { + return fmt.Errorf("rotation artifact %q does not match its recorded SHA-256 digest", artifact.Name) + } + + var observedKeyIDs []string + switch artifact.Name { + case ArtifactReplacementPublicKey: + keySet, err := jwkutil.NewSigner(payload) + if err != nil { + return fmt.Errorf("validate rotation artifact %q: %w", artifact.Name, err) + } + observedKeyIDs = []string{keySet.Keys[0].KeyID} + case ArtifactCurrentJWKS, ArtifactNewJWKS, ArtifactCombinedJWKS: + inspected, err := jwkutil.Inspect(payload) + if err != nil { + return fmt.Errorf("validate rotation artifact %q: %w", artifact.Name, err) + } + observedKeyIDs = inspected.KeyIDs + } + if !reflect.DeepEqual(observedKeyIDs, artifact.KeyIDs) { + return fmt.Errorf("rotation artifact %q key IDs do not match its recorded metadata", artifact.Name) + } + payloads[artifact.Name] = payload + } + return validateArtifactRelationships(checkpoint, payloads) +} + +func validateArtifactRelationships(checkpoint Checkpoint, payloads map[string][]byte) error { + currentRaw, hasCurrent := payloads[ArtifactCurrentJWKS] + replacementRaw, hasReplacement := payloads[ArtifactReplacementPublicKey] + if hasCurrent && checkpoint.PreRotationSignerBaseline != nil { + currentSet, err := jwkutil.Parse(currentRaw) + if err != nil { + return fmt.Errorf("validate current JWKS relationship: %w", err) + } + baselineKeyIDs := make(map[string]struct{}, len(checkpoint.PreRotationSignerBaseline.Entries)) + for _, entry := range checkpoint.PreRotationSignerBaseline.Entries { + baselineKeyIDs[entry.KeyID] = struct{}{} + } + for _, key := range currentSet.Keys { + if _, exists := baselineKeyIDs[key.KeyID]; !exists { + return fmt.Errorf("current JWKS key %q is not present in the pre-rotation public signer baseline", key.KeyID) + } + } + } + if !hasReplacement { + return nil + } + + replacementSet, err := jwkutil.NewSigner(replacementRaw) + if err != nil { + return fmt.Errorf("validate replacement signer relationship: %w", err) + } + replacementKeyID := replacementSet.Keys[0].KeyID + if checkpoint.PreRotationSignerBaseline != nil { + for _, entry := range checkpoint.PreRotationSignerBaseline.Entries { + if replacementKeyID == entry.KeyID { + return fmt.Errorf("replacement signer key %q already exists in the pre-rotation public signer baseline", replacementKeyID) + } + } + } + if checkpoint.ReplacementSigner != nil { + if replacementKeyID != checkpoint.ReplacementSigner.Entry.KeyID || publicDigest(replacementRaw) != checkpoint.ReplacementSigner.Entry.SHA256 { + return fmt.Errorf("replacement signer artifact does not match the recorded replacement evidence") + } + } + + if hasCurrent { + currentSet, err := jwkutil.Parse(currentRaw) + if err != nil { + return fmt.Errorf("validate current JWKS relationship: %w", err) + } + for _, key := range currentSet.Keys { + if key.KeyID == replacementKeyID { + return fmt.Errorf("replacement signer key %q is already present in the saved current JWKS", replacementKeyID) + } + } + } + + if newRaw, exists := payloads[ArtifactNewJWKS]; exists { + expectedNew, err := jwkutil.Encode(replacementSet) + if err != nil { + return fmt.Errorf("encode expected replacement JWKS: %w", err) + } + if !bytes.Equal(newRaw, expectedNew.Data) { + return fmt.Errorf("rotation artifact %q does not exactly represent %q", ArtifactNewJWKS, ArtifactReplacementPublicKey) + } + } + + if combinedRaw, exists := payloads[ArtifactCombinedJWKS]; exists { + if !hasCurrent { + return fmt.Errorf("rotation artifact %q requires %q", ArtifactCombinedJWKS, ArtifactCurrentJWKS) + } + currentSet, err := jwkutil.Parse(currentRaw) + if err != nil { + return fmt.Errorf("validate current JWKS relationship: %w", err) + } + expectedCombinedSet, err := jwkutil.Merge(currentSet, replacementSet) + if err != nil { + return fmt.Errorf("build expected combined JWKS: %w", err) + } + expectedCombined, err := jwkutil.Encode(expectedCombinedSet) + if err != nil { + return fmt.Errorf("encode expected combined JWKS: %w", err) + } + if !bytes.Equal(combinedRaw, expectedCombined.Data) { + return fmt.Errorf("rotation artifact %q is not the ordered union of %q and %q", ArtifactCombinedJWKS, ArtifactCurrentJWKS, ArtifactReplacementPublicKey) + } + } + + return nil +} + +func writeFileAtomically(outputDir, name string, payload []byte, mode os.FileMode) (returnErr error) { + temporary, err := os.CreateTemp(outputDir, "."+name+"-*.tmp") + if err != nil { + return fmt.Errorf("create temporary rotation file for %q: %w", name, err) + } + temporaryPath := temporary.Name() + temporaryClosed := false + defer func() { + if !temporaryClosed { + if closeErr := temporary.Close(); returnErr == nil && closeErr != nil { + returnErr = fmt.Errorf("close temporary rotation file for %q: %w", name, closeErr) + } + } + if removeErr := os.Remove(temporaryPath); returnErr == nil && removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + returnErr = fmt.Errorf("remove temporary rotation file for %q: %w", name, removeErr) + } + }() + + if err := temporary.Chmod(mode); err != nil { + return fmt.Errorf("set temporary rotation file permissions for %q: %w", name, err) + } + if _, err := temporary.Write(payload); err != nil { + return fmt.Errorf("write temporary rotation file for %q: %w", name, err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync temporary rotation file for %q: %w", name, err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary rotation file for %q: %w", name, err) + } + temporaryClosed = true + + checkpointPath := filepath.Join(outputDir, name) + if err := os.Rename(temporaryPath, checkpointPath); err != nil { + return fmt.Errorf("replace rotation file %q: %w", name, err) + } + + return syncDirectory(outputDir) +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return fmt.Errorf("open directory %q for sync: %w", path, err) + } + defer directory.Close() + if err := directory.Sync(); err != nil { + return fmt.Errorf("sync directory %q: %w", path, err) + } + + return nil +} diff --git a/pkg/cmd/provisioning/rotation/checkpoint_store_test.go b/pkg/cmd/provisioning/rotation/checkpoint_store_test.go new file mode 100644 index 0000000000..463a74f781 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/checkpoint_store_test.go @@ -0,0 +1,736 @@ +package rotation + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" +) + +func TestCheckpointStoreRoundTrip(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("SaveCheckpoint() returned unexpected error: %v", err) + } + + directoryInfo, err := os.Stat(outputDir) + if err != nil { + t.Fatalf("stat output directory: %v", err) + } + if got := directoryInfo.Mode().Perm(); got != checkpointDirMode { + t.Fatalf("output directory mode = %o, want %o", got, checkpointDirMode) + } + + checkpointPath := filepath.Join(outputDir, CheckpointFileName) + checkpointInfo, err := os.Stat(checkpointPath) + if err != nil { + t.Fatalf("stat checkpoint: %v", err) + } + if got := checkpointInfo.Mode().Perm(); got != checkpointFileMode { + t.Fatalf("checkpoint mode = %o, want %o", got, checkpointFileMode) + } + + loaded, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("LoadCheckpoint() returned unexpected error: %v", err) + } + if !filepath.IsAbs(loaded.OutputDir) { + t.Fatalf("loaded output directory %q is not absolute", loaded.OutputDir) + } + if loaded.Phase != PhaseInitialized { + t.Fatalf("loaded phase = %q, want %q", loaded.Phase, PhaseInitialized) + } + + loaded.Phase = PhasePreflightComplete + loaded.ClusterIdentity = "cluster-123" + loaded.TargetIdentity = "aws://issuer" + loaded.PreRotationSignerBaseline = testSignerBaseline() + loaded.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&loaded) + if err := SaveCheckpoint(loaded); err != nil { + t.Fatalf("SaveCheckpoint(preflight) returned unexpected error: %v", err) + } + + loaded, err = LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("LoadCheckpoint(preflight) returned unexpected error: %v", err) + } + if loaded.Phase != PhasePreflightComplete || loaded.ClusterIdentity != "cluster-123" || loaded.TargetIdentity != "aws://issuer" || !reflect.DeepEqual(loaded.PreRotationSignerBaseline, testSignerBaseline()) || !reflect.DeepEqual(loaded.PreRotationSignerRef, testSignerObjectReference()) { + t.Fatalf("loaded preflight checkpoint did not preserve identities: %#v", loaded) + } + + entries, err := os.ReadDir(outputDir) + if err != nil { + t.Fatalf("read output directory: %v", err) + } + if len(entries) != 2 || entries[0].Name() != checkpointLockFileName || entries[1].Name() != CheckpointFileName { + t.Fatalf("output directory contains unexpected files after checkpoint saves: %v", entries) + } + lockInfo, err := os.Stat(filepath.Join(outputDir, checkpointLockFileName)) + if err != nil { + t.Fatalf("stat checkpoint lock: %v", err) + } + if got := lockInfo.Mode().Perm(); got != checkpointFileMode { + t.Fatalf("checkpoint lock mode = %o, want %o", got, checkpointFileMode) + } +} + +func TestCheckpointStoreCreatesNestedDirectories(t *testing.T) { + root := t.TempDir() + first := filepath.Join(root, "first") + outputDir := filepath.Join(first, "second") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("SaveCheckpoint() returned unexpected error: %v", err) + } + + for _, directory := range []string{first, outputDir} { + info, err := os.Stat(directory) + if err != nil { + t.Fatalf("stat created directory %q: %v", directory, err) + } + if got := info.Mode().Perm(); got != checkpointDirMode { + t.Fatalf("created directory %q mode = %o, want %o", directory, got, checkpointDirMode) + } + } +} + +func TestSaveCheckpointRejectsOversizedUpdateAndPreservesReadableState(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + + checkpointPath := filepath.Join(outputDir, CheckpointFileName) + originalCheckpoint, err := os.ReadFile(checkpointPath) + if err != nil { + t.Fatalf("read original checkpoint: %v", err) + } + checkpoint.LastErrorCode = strings.Repeat("x", maxCheckpointSize) + if err := SaveCheckpoint(checkpoint); err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("SaveCheckpoint(oversized) error = %v", err) + } + + preservedCheckpoint, err := os.ReadFile(checkpointPath) + if err != nil { + t.Fatalf("read preserved checkpoint: %v", err) + } + if string(preservedCheckpoint) != string(originalCheckpoint) { + t.Fatal("checkpoint changed after oversized update") + } + if _, err := LoadCheckpoint(outputDir); err != nil { + t.Fatalf("LoadCheckpoint() after oversized update returned unexpected error: %v", err) + } +} + +func TestSaveCheckpointRejectsUnsafeTransitions(t *testing.T) { + t.Run("first checkpoint skips initialized", func(t *testing.T) { + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, filepath.Join(t.TempDir(), "output")) + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaseline() + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + err := SaveCheckpoint(checkpoint) + if err == nil || !strings.Contains(err.Error(), "first rotation checkpoint") { + t.Fatalf("SaveCheckpoint() error = %v, want first-checkpoint error", err) + } + }) + + t.Run("regression and identity change", func(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "output") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load initialized checkpoint: %v", err) + } + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaseline() + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save preflight checkpoint: %v", err) + } + + changedIdentity := checkpoint + changedIdentity.ClusterIdentity = "other-cluster" + setTestRotationGuard(&changedIdentity) + err = SaveCheckpoint(changedIdentity) + if err == nil || !strings.Contains(err.Error(), "cluster identity cannot change") { + t.Fatalf("SaveCheckpoint(changed identity) error = %v", err) + } + + changedBaseline := cloneCheckpoint(checkpoint) + changedBaseline.PreRotationSignerBaseline.Entries[0].KeyID = testOtherKeyID + setTestRotationGuard(&changedBaseline) + err = SaveCheckpoint(changedBaseline) + if err == nil || !strings.Contains(err.Error(), "public signer baseline cannot change") { + t.Fatalf("SaveCheckpoint(changed signer baseline) error = %v", err) + } + + changedSignerRef := checkpoint + changedSignerRef.PreRotationSignerRef = &SignerObjectReference{ + UID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + ResourceVersion: "54321", + } + setTestRotationGuard(&changedSignerRef) + err = SaveCheckpoint(changedSignerRef) + if err == nil || !strings.Contains(err.Error(), "signer object reference cannot change") { + t.Fatalf("SaveCheckpoint(changed signer reference) error = %v", err) + } + + changedGuard := cloneCheckpoint(checkpoint) + changedGuard.RotationGuard.OperationID = strings.Repeat("f", 64) + err = validateCheckpointTransition(checkpoint, changedGuard) + if err == nil || !strings.Contains(err.Error(), "guard reference cannot change") { + t.Fatalf("validateCheckpointTransition(changed guard) error = %v", err) + } + + regressed := checkpoint + regressed.Phase = PhaseInitialized + err = validateCheckpointTransition(checkpoint, regressed) + if err == nil || !strings.Contains(err.Error(), "cannot regress") { + t.Fatalf("validateCheckpointTransition(regression) error = %v", err) + } + }) + + t.Run("phase skip", func(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "output") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + + currentData, currentMetadata := currentJWKSForTest(t) + if err := os.WriteFile(filepath.Join(outputDir, ArtifactCurrentJWKS), currentData, 0o644); err != nil { + t.Fatalf("write current JWKS: %v", err) + } + checkpoint.OutputDir = outputDir + checkpoint.Phase = PhaseCurrentJWKSRead + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaselineForKeyID(currentMetadata.KeyIDs[0]) + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + checkpoint.Artifacts = []ArtifactMetadata{currentMetadata} + err := SaveCheckpoint(checkpoint) + if err == nil || !strings.Contains(err.Error(), "cannot skip") { + t.Fatalf("SaveCheckpoint(skip) error = %v", err) + } + }) +} + +func TestLoadCheckpointRejectsUnsafeInput(t *testing.T) { + tests := []struct { + name string + payload func(string) []byte + mode os.FileMode + wantError string + }{ + { + name: "unknown field", + payload: func(outputDir string) []byte { + return []byte(fmt.Sprintf(`{"schemaVersion":1,"provider":"aws","publicationMode":"direct","phase":"initialized","outputDir":%q,"unexpected":true}`, outputDir)) + }, + mode: 0o600, + wantError: "unknown field", + }, + { + name: "trailing JSON", + payload: func(outputDir string) []byte { + return []byte(fmt.Sprintf(`{"schemaVersion":1,"provider":"aws","publicationMode":"direct","phase":"initialized","outputDir":%q} {}`, outputDir)) + }, + mode: 0o600, + wantError: "trailing JSON value", + }, + { + name: "malformed JSON", + payload: func(string) []byte { + return []byte(`{"schemaVersion":`) + }, + mode: 0o600, + wantError: "decode rotation checkpoint", + }, + { + name: "oversized checkpoint", + payload: func(string) []byte { + return []byte(strings.Repeat(" ", maxCheckpointSize+1)) + }, + mode: 0o600, + wantError: "exceeds", + }, + { + name: "insecure permissions", + payload: func(outputDir string) []byte { + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + payload, err := json.Marshal(checkpoint) + if err != nil { + t.Fatalf("marshal checkpoint: %v", err) + } + return payload + }, + mode: 0o644, + wantError: "permissions", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + outputDir := t.TempDir() + checkpointPath := filepath.Join(outputDir, CheckpointFileName) + if err := os.WriteFile(checkpointPath, test.payload(outputDir), test.mode); err != nil { + t.Fatalf("write checkpoint fixture: %v", err) + } + if err := os.Chmod(checkpointPath, test.mode); err != nil { + t.Fatalf("chmod checkpoint fixture: %v", err) + } + + _, err := LoadCheckpoint(outputDir) + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("LoadCheckpoint() error = %v, want error containing %q", err, test.wantError) + } + }) + } + + t.Run("symlink", func(t *testing.T) { + outputDir := t.TempDir() + target := filepath.Join(outputDir, "target.json") + if err := os.WriteFile(target, []byte(`{}`), 0o600); err != nil { + t.Fatalf("write symlink target: %v", err) + } + if err := os.Symlink(target, filepath.Join(outputDir, CheckpointFileName)); err != nil { + t.Fatalf("create checkpoint symlink: %v", err) + } + _, err := LoadCheckpoint(outputDir) + if err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("LoadCheckpoint(symlink) error = %v", err) + } + }) + + t.Run("group-writable output directory", func(t *testing.T) { + outputDir := t.TempDir() + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + payload, err := json.Marshal(checkpoint) + if err != nil { + t.Fatalf("marshal checkpoint: %v", err) + } + if err := os.WriteFile(filepath.Join(outputDir, CheckpointFileName), payload, checkpointFileMode); err != nil { + t.Fatalf("write checkpoint fixture: %v", err) + } + if err := os.Chmod(outputDir, 0o770); err != nil { + t.Fatalf("make output directory group-writable: %v", err) + } + + _, err = LoadCheckpoint(outputDir) + if err == nil || !strings.Contains(err.Error(), "must not be writable") { + t.Fatalf("LoadCheckpoint(writable output directory) error = %v", err) + } + }) +} + +func TestLoadCheckpointRejectsArtifactTampering(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "output") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load initialized checkpoint: %v", err) + } + currentData, currentMetadata := currentJWKSForTest(t) + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaselineForKeyID(currentMetadata.KeyIDs[0]) + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save preflight checkpoint: %v", err) + } + checkpoint.Phase = PhaseGuardAcquired + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save guard-acquired checkpoint: %v", err) + } + + if err := os.WriteFile(filepath.Join(outputDir, ArtifactCurrentJWKS), currentData, 0o644); err != nil { + t.Fatalf("write current JWKS: %v", err) + } + checkpoint.Phase = PhaseCurrentJWKSRead + checkpoint.Artifacts = []ArtifactMetadata{currentMetadata} + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save current-JWKS checkpoint: %v", err) + } + + if err := os.WriteFile(filepath.Join(outputDir, ArtifactCurrentJWKS), []byte("tampered"), 0o644); err != nil { + t.Fatalf("tamper with current JWKS: %v", err) + } + _, err = LoadCheckpoint(outputDir) + if err == nil || !strings.Contains(err.Error(), "does not match its recorded") { + t.Fatalf("LoadCheckpoint(tampered artifact) error = %v", err) + } +} + +func TestSaveCheckpointDoesNotResetStateWhenRecordedArtifactIsMissing(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "output") + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save initialized checkpoint: %v", err) + } + + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load initialized checkpoint: %v", err) + } + currentData, currentMetadata := currentJWKSForTest(t) + checkpoint.Phase = PhasePreflightComplete + checkpoint.ClusterIdentity = "cluster-123" + checkpoint.TargetIdentity = "aws://issuer" + checkpoint.PreRotationSignerBaseline = testSignerBaselineForKeyID(currentMetadata.KeyIDs[0]) + checkpoint.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&checkpoint) + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save preflight checkpoint: %v", err) + } + checkpoint.Phase = PhaseGuardAcquired + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save guard-acquired checkpoint: %v", err) + } + + if err := os.WriteFile(filepath.Join(outputDir, ArtifactCurrentJWKS), currentData, artifactFileMode); err != nil { + t.Fatalf("write current JWKS: %v", err) + } + checkpoint.Phase = PhaseCurrentJWKSRead + checkpoint.Artifacts = []ArtifactMetadata{currentMetadata} + if err := SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("save current-JWKS checkpoint: %v", err) + } + + checkpointPath := filepath.Join(outputDir, CheckpointFileName) + originalCheckpoint, err := os.ReadFile(checkpointPath) + if err != nil { + t.Fatalf("read original checkpoint: %v", err) + } + if err := os.Remove(filepath.Join(outputDir, ArtifactCurrentJWKS)); err != nil { + t.Fatalf("remove recorded current JWKS: %v", err) + } + + reset := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + err = SaveCheckpoint(reset) + if err == nil || !strings.Contains(err.Error(), "validate recorded rotation artifact") { + t.Fatalf("SaveCheckpoint(reset with missing artifact) error = %v", err) + } + preservedCheckpoint, err := os.ReadFile(checkpointPath) + if err != nil { + t.Fatalf("read preserved checkpoint: %v", err) + } + if string(preservedCheckpoint) != string(originalCheckpoint) { + t.Fatal("checkpoint changed after recorded artifact was lost") + } +} + +func TestValidateArtifactRelationships(t *testing.T) { + currentPEM := testPublicKeyPEM(t) + replacementPEM := testPublicKeyPEM(t) + unrelatedPEM := testPublicKeyPEM(t) + currentRaw := encodedJWKSForTest(t, currentPEM) + valid, err := PrepareJWKSArtifacts(currentRaw, replacementPEM) + if err != nil { + t.Fatalf("prepare valid artifact bundle: %v", err) + } + unrelatedNew := encodedJWKSForTest(t, unrelatedPEM) + unrelatedBundle, err := PrepareJWKSArtifacts(currentRaw, unrelatedPEM) + if err != nil { + t.Fatalf("prepare unrelated artifact bundle: %v", err) + } + replacementSet, err := jwkutil.NewSigner(replacementPEM) + if err != nil { + t.Fatalf("derive replacement key ID: %v", err) + } + replacementKeyID := replacementSet.Keys[0].KeyID + + validPayloads := map[string][]byte{ + ArtifactCurrentJWKS: valid.Current.Data, + ArtifactReplacementPublicKey: replacementPEM, + ArtifactNewJWKS: valid.New.Data, + ArtifactCombinedJWKS: valid.Combined.Data, + } + + tests := []struct { + name string + baselineKeyIDs []string + payloads map[string][]byte + wantError string + }{ + {name: "valid bundle", payloads: validPayloads}, + { + name: "replacement without generated artifacts", + payloads: map[string][]byte{ + ArtifactCurrentJWKS: valid.Current.Data, + ArtifactReplacementPublicKey: replacementPEM, + }, + }, + { + name: "replacement already in current", + baselineKeyIDs: []string{ + valid.Current.KeyIDs[0], + replacementKeyID, + }, + payloads: map[string][]byte{ + ArtifactCurrentJWKS: valid.Combined.Data, + ArtifactReplacementPublicKey: replacementPEM, + }, + wantError: "already exists", + }, + { + name: "replacement did not change", + baselineKeyIDs: []string{replacementKeyID}, + payloads: map[string][]byte{ + ArtifactCurrentJWKS: valid.New.Data, + ArtifactReplacementPublicKey: replacementPEM, + }, + wantError: "pre-rotation public signer baseline", + }, + { + name: "new JWKS uses unrelated key", + payloads: map[string][]byte{ + ArtifactCurrentJWKS: valid.Current.Data, + ArtifactReplacementPublicKey: replacementPEM, + ArtifactNewJWKS: unrelatedNew, + }, + wantError: "does not exactly represent", + }, + { + name: "new JWKS is not canonical", + payloads: map[string][]byte{ + ArtifactCurrentJWKS: valid.Current.Data, + ArtifactReplacementPublicKey: replacementPEM, + ArtifactNewJWKS: append([]byte(" \n"), valid.New.Data...), + }, + wantError: "does not exactly represent", + }, + { + name: "combined JWKS uses unrelated replacement", + payloads: map[string][]byte{ + ArtifactCurrentJWKS: valid.Current.Data, + ArtifactReplacementPublicKey: replacementPEM, + ArtifactNewJWKS: valid.New.Data, + ArtifactCombinedJWKS: unrelatedBundle.Combined.Data, + }, + wantError: "is not the ordered union", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + baselineKeyIDs := test.baselineKeyIDs + if len(baselineKeyIDs) == 0 { + baselineKeyIDs = []string{valid.Current.KeyIDs[0]} + } + checkpoint := Checkpoint{PreRotationSignerBaseline: testSignerBaselineForKeyIDs(baselineKeyIDs...)} + err := validateArtifactRelationships(checkpoint, test.payloads) + if test.wantError == "" { + if err != nil { + t.Fatalf("validateArtifactRelationships() returned unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("validateArtifactRelationships() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestValidateCheckpointTransitionPreservesEvidence(t *testing.T) { + previous := completeTestCheckpoint() + + tests := []struct { + name string + mutate func(*Checkpoint) + wantError string + }{ + { + name: "artifact removed", + mutate: func(next *Checkpoint) { + next.Artifacts = next.Artifacts[1:] + }, + wantError: "cannot remove recorded artifact", + }, + { + name: "artifact metadata changed", + mutate: func(next *Checkpoint) { + next.Artifacts[0].KeyIDs = []string{"changed"} + }, + wantError: "cannot change recorded artifact", + }, + { + name: "publication removed", + mutate: func(next *Checkpoint) { + next.Publications = next.Publications[1:] + }, + wantError: "cannot remove publication confirmation", + }, + { + name: "publication changed", + mutate: func(next *Checkpoint) { + next.Publications[0].SHA256 = strings.Repeat("b", 64) + }, + wantError: "cannot change publication confirmation", + }, + { + name: "pre-rotation signer baseline changed", + mutate: func(next *Checkpoint) { + next.PreRotationSignerBaseline.Entries[0].SHA256 = strings.Repeat("c", 64) + }, + wantError: "public signer baseline cannot change", + }, + { + name: "pre-rotation signer reference changed", + mutate: func(next *Checkpoint) { + next.PreRotationSignerRef.ResourceVersion = "54321" + }, + wantError: "signer object reference cannot change", + }, + { + name: "replacement signer evidence changed", + mutate: func(next *Checkpoint) { + next.ReplacementSigner.SecretRef.ResourceVersion = "54321" + }, + wantError: "replacement signer evidence cannot change", + }, + { + name: "reboot intent removed", + mutate: func(next *Checkpoint) { + next.RebootIntent = nil + }, + wantError: "reboot intent cannot change", + }, + { + name: "reboot intent baseline changed", + mutate: func(next *Checkpoint) { + next.RebootIntent.Baselines[0].BootID = "changed-boot-id" + }, + wantError: "reboot intent cannot change", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + next := cloneCheckpoint(previous) + test.mutate(&next) + err := validateCheckpointTransition(previous, next) + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("validateCheckpointTransition() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestValidateCheckpointTransitionRequiresDurableRebootIntentPhase(t *testing.T) { + stable := completeTestCheckpoint() + stable.Phase = PhaseSignerRolloutStable + stable.Publications = stable.Publications[:1] + stable.RebootIntent = nil + if err := stable.Validate(); err != nil { + t.Fatalf("signer-rollout fixture is invalid: %v", err) + } + + intentRecorded := stable + intentRecorded.Phase = PhaseRebootIntentRecorded + intentRecorded.RebootIntent = testNodeRebootIntent() + if err := intentRecorded.Validate(); err != nil { + t.Fatalf("reboot-intent fixture is invalid: %v", err) + } + if err := validateCheckpointTransition(stable, intentRecorded); err != nil { + t.Fatalf("signer rollout to reboot intent transition failed: %v", err) + } + + nodesRebooted := intentRecorded + nodesRebooted.Phase = PhaseNodesRebooted + if err := nodesRebooted.Validate(); err != nil { + t.Fatalf("nodes-rebooted fixture is invalid: %v", err) + } + if err := validateCheckpointTransition(stable, nodesRebooted); err == nil || !strings.Contains(err.Error(), "cannot skip") { + t.Fatalf("direct signer rollout to nodes rebooted transition error = %v, want phase-skip error", err) + } +} + +func TestValidateCheckpointTransitionAllowsOnlyExplicitCanonicalRebootAdoption(t *testing.T) { + previous := completeTestCheckpoint() + previous.Phase = PhaseRebootIntentRecorded + previous.Publications = previous.Publications[:1] + + next := previous + next.Artifacts = append([]ArtifactMetadata(nil), previous.Artifacts...) + next.Publications = append([]PublicationConfirmation(nil), previous.Publications...) + signerReference := *previous.PreRotationSignerRef + next.PreRotationSignerRef = &signerReference + canonical := cloneRebootIntent(*previous.RebootIntent) + canonical.Targets = []string{"worker"} + canonical.Baselines = []NodeRebootBaseline{{Target: "worker", Node: "worker-0", BootID: "cluster-canonical-boot-id"}} + next.RebootIntent = &canonical + + if err := validateCheckpointTransition(previous, next); err == nil || !strings.Contains(err.Error(), "reboot intent cannot change") { + t.Fatalf("ordinary transition error = %v, want immutable reboot intent rejection", err) + } + if err := validateCheckpointTransitionWithCanonicalRebootIntentAdoption(previous, next, true); err != nil { + t.Fatalf("explicit canonical reboot adoption failed: %v", err) + } + + differentID := next + differentIntent := cloneRebootIntent(*next.RebootIntent) + differentIntent.ID = "different-operation" + differentID.RebootIntent = &differentIntent + if err := validateCheckpointTransitionWithCanonicalRebootIntentAdoption(previous, differentID, true); err == nil || !strings.Contains(err.Error(), "reboot intent cannot change") { + t.Fatalf("different-ID adoption error = %v, want rejection", err) + } + + afterAdvance := next + afterAdvance.Phase = PhaseNodesRebooted + if err := validateCheckpointTransitionWithCanonicalRebootIntentAdoption(previous, afterAdvance, true); err == nil || !strings.Contains(err.Error(), "reboot intent cannot change") { + t.Fatalf("post-advance adoption error = %v, want rejection", err) + } +} + +func currentJWKSForTest(t *testing.T) ([]byte, ArtifactMetadata) { + t.Helper() + artifactData := encodedJWKSForTest(t, testPublicKeyPEM(t)) + artifact, err := jwkutil.Inspect(artifactData) + if err != nil { + t.Fatalf("inspect current JWKS: %v", err) + } + return artifact.Data, ArtifactMetadata{ + Name: ArtifactCurrentJWKS, + SHA256: artifact.SHA256, + KeyIDs: artifact.KeyIDs, + } +} + +func encodedJWKSForTest(t *testing.T, publicPEM []byte) []byte { + t.Helper() + keySet, err := jwkutil.NewSigner(publicPEM) + if err != nil { + t.Fatalf("create current JWKS: %v", err) + } + artifact, err := jwkutil.Encode(keySet) + if err != nil { + t.Fatalf("encode current JWKS: %v", err) + } + return artifact.Data +} diff --git a/pkg/cmd/provisioning/rotation/command.go b/pkg/cmd/provisioning/rotation/command.go new file mode 100644 index 0000000000..911b599f1e --- /dev/null +++ b/pkg/cmd/provisioning/rotation/command.go @@ -0,0 +1,112 @@ +package rotation + +import ( + "context" + "fmt" + "strings" + + "github.com/spf13/cobra" +) + +// Runner executes one new or resumed rotation operation. +type Runner interface { + Run(context.Context, RunOptions) (RunResult, error) +} + +// RunnerSetup contains the concrete runner and any provider-owned manual input +// collected by its command wrapper. +type RunnerSetup struct { + Runner Runner + ManualInput ManualInput +} + +// RunnerFactory constructs the provider and cluster adapters for one explicit +// kubeconfig and a validated set of common run options. Provider wrappers may +// add their own manual-input or acknowledgement flags and capture those values +// in this factory; the shared command intentionally does not declare them. +type RunnerFactory func(context.Context, string, RunOptions) (RunnerSetup, error) + +type commandOptions struct { + kubeconfig string + outputDir string + publicationMode string + resume bool +} + +// NewCommand returns the provider-neutral signing-key rotation command. It is +// deliberately not registered by this package; provider packages own that +// integration after constructing their concrete adapters. Provider wrappers +// may add provider-owned flags and capture their values in runnerFactory. +func NewCommand(provider Provider, runnerFactory RunnerFactory) *cobra.Command { + options := commandOptions{ + publicationMode: string(PublicationModeDirect), + } + + command := &cobra.Command{ + Use: "rotate-signing-key", + Short: "Rotate the bound service account signing key", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + return options.run(command.Context(), provider, runnerFactory) + }, + } + + flags := command.Flags() + flags.StringVar(&options.kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file") + flags.StringVar(&options.outputDir, "output-dir", "", "Directory for public rotation artifacts and checkpoint state") + flags.StringVar(&options.publicationMode, "publication-mode", string(PublicationModeDirect), "JWKS publication mode: direct or manual") + flags.BoolVar(&options.resume, "resume", false, "Resume the rotation recorded in the output directory") + mustMarkRequired(command, "kubeconfig") + mustMarkRequired(command, "output-dir") + + return command +} + +func (o commandOptions) run(ctx context.Context, provider Provider, runnerFactory RunnerFactory) error { + if strings.TrimSpace(o.kubeconfig) == "" { + return fmt.Errorf("kubeconfig must not be empty") + } + if strings.TrimSpace(o.outputDir) == "" { + return fmt.Errorf("output directory must not be empty") + } + if !isSupportedProvider(provider) { + return fmt.Errorf("unsupported rotation provider %q", provider) + } + publicationMode := PublicationMode(o.publicationMode) + if !isSupportedPublicationMode(publicationMode) { + return fmt.Errorf("unsupported rotation publication mode %q", o.publicationMode) + } + if runnerFactory == nil { + return fmt.Errorf("rotation runner factory must not be nil") + } + + runOptions := RunOptions{ + Provider: provider, + PublicationMode: publicationMode, + OutputDir: o.outputDir, + Resume: o.resume, + } + setup, err := runnerFactory(ctx, o.kubeconfig, runOptions) + if err != nil { + return fmt.Errorf("construct rotation runner: %w", err) + } + if setup.Runner == nil { + return fmt.Errorf("rotation runner factory returned a nil runner") + } + + runOptions.Manual = setup.ManualInput + result, err := setup.Runner.Run(ctx, runOptions) + if err != nil { + return err + } + if !result.Complete || result.Phase != PhaseComplete { + return fmt.Errorf("rotation runner returned incomplete result at phase %q", result.Phase) + } + return nil +} + +func mustMarkRequired(command *cobra.Command, flagName string) { + if err := command.MarkFlagRequired(flagName); err != nil { + panic(err) + } +} diff --git a/pkg/cmd/provisioning/rotation/command_test.go b/pkg/cmd/provisioning/rotation/command_test.go new file mode 100644 index 0000000000..18f764402b --- /dev/null +++ b/pkg/cmd/provisioning/rotation/command_test.go @@ -0,0 +1,308 @@ +package rotation + +import ( + "context" + "errors" + "io" + "reflect" + "sort" + "strings" + "testing" + + "github.com/spf13/pflag" +) + +func TestNewCommandPassesValidatedOptionsAndManualInputToRunner(t *testing.T) { + t.Parallel() + + type contextKey string + const key contextKey = "command-test" + ctx := context.WithValue(context.Background(), key, "context-value") + + runner := &commandTestRunner{result: completeCommandRunResult()} + manualInput := ManualInput{CurrentJWKS: []byte(`{"keys":[]}`)} + var factoryContext context.Context + var factoryKubeconfig string + var factoryOptions RunOptions + command := NewCommand(ProviderAzure, func(ctx context.Context, kubeconfig string, options RunOptions) (RunnerSetup, error) { + factoryContext = ctx + factoryKubeconfig = kubeconfig + factoryOptions = options + return RunnerSetup{Runner: runner, ManualInput: manualInput}, nil + }) + command.SetOut(io.Discard) + command.SetErr(io.Discard) + command.SetArgs([]string{ + "--kubeconfig", "/tmp/cluster.kubeconfig", + "--output-dir", "/tmp/rotation-state", + "--publication-mode", "manual", + "--resume", + }) + + if err := command.ExecuteContext(ctx); err != nil { + t.Fatalf("ExecuteContext() returned unexpected error: %v", err) + } + if factoryContext != ctx { + t.Fatal("runner factory did not receive the command context") + } + if factoryKubeconfig != "/tmp/cluster.kubeconfig" { + t.Fatalf("factory kubeconfig = %q, want %q", factoryKubeconfig, "/tmp/cluster.kubeconfig") + } + if runner.runContext != ctx { + t.Fatal("runner did not receive the command context") + } + if got := runner.runContext.Value(key); got != "context-value" { + t.Fatalf("runner context value = %v, want context-value", got) + } + wantBaseOptions := RunOptions{ + Provider: ProviderAzure, + PublicationMode: PublicationModeManual, + OutputDir: "/tmp/rotation-state", + Resume: true, + } + if !reflect.DeepEqual(factoryOptions, wantBaseOptions) { + t.Fatalf("factory options = %+v, want %+v", factoryOptions, wantBaseOptions) + } + wantRunnerOptions := wantBaseOptions + wantRunnerOptions.Manual = manualInput + if !reflect.DeepEqual(runner.options, wantRunnerOptions) { + t.Fatalf("runner options = %+v, want %+v", runner.options, wantRunnerOptions) + } + if runner.calls != 1 { + t.Fatalf("runner calls = %d, want 1", runner.calls) + } +} + +func TestNewCommandValidatesArgumentsAndCommonFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + provider Provider + args []string + factory RunnerFactory + wantError string + }{ + { + name: "requires kubeconfig", + provider: ProviderAWS, + args: []string{"--output-dir", "/tmp/output"}, + factory: successfulCommandRunnerFactory(), + wantError: "required flag(s) \"kubeconfig\" not set", + }, + { + name: "requires output directory", + provider: ProviderAWS, + args: []string{"--kubeconfig", "/tmp/kubeconfig"}, + factory: successfulCommandRunnerFactory(), + wantError: "required flag(s) \"output-dir\" not set", + }, + { + name: "rejects explicitly empty kubeconfig", + provider: ProviderAWS, + args: []string{"--kubeconfig=", "--output-dir", "/tmp/output"}, + factory: successfulCommandRunnerFactory(), + wantError: "kubeconfig must not be empty", + }, + { + name: "rejects explicitly empty output directory", + provider: ProviderAWS, + args: []string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir="}, + factory: successfulCommandRunnerFactory(), + wantError: "output directory must not be empty", + }, + { + name: "rejects positional arguments", + provider: ProviderAWS, + args: []string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output", "unexpected"}, + factory: successfulCommandRunnerFactory(), + wantError: "unknown command \"unexpected\" for \"rotate-signing-key\"", + }, + { + name: "rejects unsupported publication mode", + provider: ProviderAWS, + args: []string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output", "--publication-mode", "automatic"}, + factory: successfulCommandRunnerFactory(), + wantError: "unsupported rotation publication mode \"automatic\"", + }, + { + name: "rejects unsupported provider", + provider: Provider("unsupported"), + args: []string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}, + factory: successfulCommandRunnerFactory(), + wantError: "unsupported rotation provider \"unsupported\"", + }, + { + name: "rejects nil runner factory", + provider: ProviderAWS, + args: []string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}, + wantError: "rotation runner factory must not be nil", + }, + { + name: "rejects nil runner", + provider: ProviderAWS, + args: []string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}, + factory: func(context.Context, string, RunOptions) (RunnerSetup, error) { + return RunnerSetup{}, nil + }, + wantError: "rotation runner factory returned a nil runner", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + command := NewCommand(test.provider, test.factory) + command.SilenceErrors = true + command.SilenceUsage = true + command.SetOut(io.Discard) + command.SetErr(io.Discard) + command.SetArgs(test.args) + + err := command.Execute() + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("Execute() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestNewCommandHasOnlyCommonFlagsAndIndependentDefaults(t *testing.T) { + t.Parallel() + + firstRunner := &commandTestRunner{result: completeCommandRunResult()} + first := NewCommand(ProviderGCP, func(context.Context, string, RunOptions) (RunnerSetup, error) { + return RunnerSetup{Runner: firstRunner}, nil + }) + first.SetOut(io.Discard) + first.SetErr(io.Discard) + first.SetArgs([]string{ + "--kubeconfig", "/tmp/first-kubeconfig", + "--output-dir", "/tmp/first-output", + "--publication-mode", "manual", + "--resume", + }) + if err := first.Execute(); err != nil { + t.Fatalf("first Execute() returned unexpected error: %v", err) + } + + secondRunner := &commandTestRunner{result: completeCommandRunResult()} + second := NewCommand(ProviderGCP, func(context.Context, string, RunOptions) (RunnerSetup, error) { + return RunnerSetup{Runner: secondRunner}, nil + }) + second.SetOut(io.Discard) + second.SetErr(io.Discard) + second.SetArgs([]string{ + "--kubeconfig", "/tmp/second-kubeconfig", + "--output-dir", "/tmp/second-output", + }) + if err := second.Execute(); err != nil { + t.Fatalf("second Execute() returned unexpected error: %v", err) + } + + if secondRunner.options.PublicationMode != PublicationModeDirect { + t.Fatalf("second command publication mode = %q, want default %q", secondRunner.options.PublicationMode, PublicationModeDirect) + } + if secondRunner.options.Resume { + t.Fatal("second command inherited resume from the first command") + } + + var flagNames []string + second.LocalNonPersistentFlags().VisitAll(func(flag *pflag.Flag) { + if flag.Name == "help" { + return + } + flagNames = append(flagNames, flag.Name) + }) + sort.Strings(flagNames) + wantFlagNames := []string{"kubeconfig", "output-dir", "publication-mode", "resume"} + if !reflect.DeepEqual(flagNames, wantFlagNames) { + t.Fatalf("local flags = %v, want %v", flagNames, wantFlagNames) + } +} + +func TestNewCommandPropagatesFactoryAndRunnerErrors(t *testing.T) { + t.Parallel() + + factoryError := errors.New("factory failed") + factoryCommand := NewCommand(ProviderAWS, func(context.Context, string, RunOptions) (RunnerSetup, error) { + return RunnerSetup{}, factoryError + }) + factoryCommand.SilenceErrors = true + factoryCommand.SilenceUsage = true + factoryCommand.SetArgs([]string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}) + if err := factoryCommand.Execute(); !errors.Is(err, factoryError) { + t.Fatalf("factory command error = %v, want %v", err, factoryError) + } + + runnerError := errors.New("runner failed") + runnerCommand := NewCommand(ProviderAWS, func(context.Context, string, RunOptions) (RunnerSetup, error) { + return RunnerSetup{Runner: &commandTestRunner{err: runnerError}}, nil + }) + runnerCommand.SilenceErrors = true + runnerCommand.SilenceUsage = true + runnerCommand.SetArgs([]string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}) + if err := runnerCommand.Execute(); !errors.Is(err, runnerError) { + t.Fatalf("runner command error = %v, want %v", err, runnerError) + } +} + +func TestNewCommandRejectsIncompleteRunnerResult(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + result RunResult + }{ + { + name: "incomplete flag at complete phase", + result: RunResult{Phase: PhaseComplete, Complete: false}, + }, + { + name: "complete flag before complete phase", + result: RunResult{Phase: PhasePostRebootStable, Complete: true}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + command := NewCommand(ProviderAWS, func(context.Context, string, RunOptions) (RunnerSetup, error) { + return RunnerSetup{Runner: &commandTestRunner{result: test.result}}, nil + }) + command.SilenceErrors = true + command.SilenceUsage = true + command.SetArgs([]string{"--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}) + + err := command.Execute() + if err == nil || !strings.Contains(err.Error(), "rotation runner returned incomplete result") { + t.Fatalf("Execute() error = %v, want incomplete-result error", err) + } + }) + } +} + +type commandTestRunner struct { + runContext context.Context + options RunOptions + calls int + result RunResult + err error +} + +func (r *commandTestRunner) Run(ctx context.Context, options RunOptions) (RunResult, error) { + r.runContext = ctx + r.options = options + r.calls++ + return r.result, r.err +} + +func successfulCommandRunnerFactory() RunnerFactory { + return func(context.Context, string, RunOptions) (RunnerSetup, error) { + return RunnerSetup{Runner: &commandTestRunner{result: completeCommandRunResult()}}, nil + } +} + +func completeCommandRunResult() RunResult { + return RunResult{Phase: PhaseComplete, Complete: true} +} diff --git a/pkg/cmd/provisioning/rotation/contract.go b/pkg/cmd/provisioning/rotation/contract.go new file mode 100644 index 0000000000..0b1ec3c188 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/contract.go @@ -0,0 +1,634 @@ +package rotation + +import ( + "encoding/base64" + "encoding/hex" + "fmt" + "regexp" + "strings" + "unicode" +) + +const CheckpointSchemaVersion = 1 + +var publicSignerEntryNamePattern = regexp.MustCompile(`^service-account-[0-9]+\.pub$`) + +type Provider string + +const ( + ProviderAWS Provider = "aws" + ProviderAzure Provider = "azure" + ProviderGCP Provider = "gcp" +) + +type PublicationMode string + +const ( + PublicationModeDirect PublicationMode = "direct" + PublicationModeManual PublicationMode = "manual" +) + +const ( + ArtifactReplacementPublicKey = "serviceaccount-signer.public" + ArtifactCurrentJWKS = "jwks.current.json" + ArtifactNewJWKS = "jwks.new.json" + ArtifactCombinedJWKS = "jwks.combined.json" +) + +type Phase string + +const ( + PhaseInitialized Phase = "initialized" + PhasePreflightComplete Phase = "preflight-complete" + PhaseGuardAcquired Phase = "guard-acquired" + PhaseCurrentJWKSRead Phase = "current-jwks-read" + PhaseNextKeyRequested Phase = "next-key-requested" + PhaseNextPublicKeyRead Phase = "next-public-key-read" + PhaseNewJWKSBuilt Phase = "new-jwks-built" + PhaseCombinedJWKSBuilt Phase = "combined-jwks-built" + PhaseCombinedJWKSPublished Phase = "combined-jwks-published" + PhaseSignerRolloutStable Phase = "signer-rollout-stable" + PhaseRebootIntentRecorded Phase = "reboot-intent-recorded" + PhaseNodesRebooted Phase = "nodes-rebooted" + PhasePostRebootStable Phase = "post-reboot-stable" + PhaseNewOnlyJWKSPublished Phase = "new-only-jwks-published" + PhaseGuardReleaseRecorded Phase = "guard-release-recorded" + PhaseComplete Phase = "complete" +) + +var orderedPhases = []Phase{ + PhaseInitialized, + PhasePreflightComplete, + PhaseGuardAcquired, + PhaseCurrentJWKSRead, + PhaseNextKeyRequested, + PhaseNextPublicKeyRead, + PhaseNewJWKSBuilt, + PhaseCombinedJWKSBuilt, + PhaseCombinedJWKSPublished, + PhaseSignerRolloutStable, + PhaseRebootIntentRecorded, + PhaseNodesRebooted, + PhasePostRebootStable, + PhaseNewOnlyJWKSPublished, + PhaseGuardReleaseRecorded, + PhaseComplete, +} + +type ArtifactMetadata struct { + Name string `json:"name"` + SHA256 string `json:"sha256,omitempty"` + KeyIDs []string `json:"keyIDs,omitempty"` +} + +// PublicationConfirmation binds a completed publication checkpoint to the +// exact public artifact that was applied. Resuming code must still reconcile +// the provider's observable state; this local record is not proof by itself. +type PublicationConfirmation struct { + Phase Phase `json:"phase"` + Artifact string `json:"artifact"` + SHA256 string `json:"sha256"` +} + +// SignerObjectReference identifies the pre-rotation signer Secret without +// recording any Secret payload. Both values are used as deletion preconditions +// so a delayed retry cannot delete a replacement object. +type SignerObjectReference struct { + UID string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` +} + +// PublicSignerBaselineEntry binds one public signer ConfigMap entry name to +// both its exact byte representation and its derived key identity. Recording +// names and digests prevents an overwrite or rename from being mistaken for a +// newly appended signer. +type PublicSignerBaselineEntry struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + KeyID string `json:"keyID"` +} + +// PublicSignerBaseline records the complete public signer ConfigMap before +// rotation. The resource version is point-in-time evidence; after replacement +// the ConfigMap is expected to have a newer resource version but the UID and +// every recorded entry must remain unchanged. +type PublicSignerBaseline struct { + ConfigMapUID string `json:"configMapUID"` + ConfigMapResourceVersion string `json:"configMapResourceVersion"` + Entries []PublicSignerBaselineEntry `json:"entries"` +} + +// ReplacementSignerEvidence binds the one appended public ConfigMap entry to +// the metadata-only identity of the replacement signer Secret. +type ReplacementSignerEvidence struct { + Entry PublicSignerBaselineEntry `json:"entry"` + SecretRef SignerObjectReference `json:"secretRef"` +} + +// RotationGuardReference deterministically identifies both the cluster-wide +// signer-rotation guard scope and one exact operation. It contains no secret +// material and is derived from immutable preflight evidence. +type RotationGuardReference struct { + ScopeID string `json:"scopeID"` + OperationID string `json:"operationID"` +} + +// NodeRebootBaseline records the observable, non-secret state used to prove +// that a node in a target machine config pool rebooted after the intent was +// persisted. +type NodeRebootBaseline struct { + Target string `json:"target"` + Node string `json:"node"` + BootID string `json:"bootID"` +} + +// RebootIntent durably identifies one disruptive reboot request and the exact +// machine config pools and node boot IDs that must be reconciled on resume. +type RebootIntent struct { + ID string `json:"id"` + Targets []string `json:"targets"` + Baselines []NodeRebootBaseline `json:"baselines"` +} + +// Checkpoint records only the non-secret information needed to validate and +// resume an externally initiated rotation. Cloud credentials and signer key +// material must never be added to this structure. +type Checkpoint struct { + SchemaVersion int `json:"schemaVersion"` + Provider Provider `json:"provider"` + PublicationMode PublicationMode `json:"publicationMode"` + Phase Phase `json:"phase"` + OutputDir string `json:"outputDir"` + ClusterIdentity string `json:"clusterIdentity,omitempty"` + TargetIdentity string `json:"targetIdentity,omitempty"` + PreRotationSignerBaseline *PublicSignerBaseline `json:"preRotationSignerBaseline,omitempty"` + PreRotationSignerRef *SignerObjectReference `json:"preRotationSignerRef,omitempty"` + RotationGuard *RotationGuardReference `json:"rotationGuard,omitempty"` + ReplacementSigner *ReplacementSignerEvidence `json:"replacementSigner,omitempty"` + RebootIntent *RebootIntent `json:"rebootIntent,omitempty"` + Artifacts []ArtifactMetadata `json:"artifacts,omitempty"` + Publications []PublicationConfirmation `json:"publicationConfirmations,omitempty"` + LastErrorCode string `json:"lastErrorCode,omitempty"` +} + +func NewCheckpoint(provider Provider, publicationMode PublicationMode, outputDir string) Checkpoint { + return Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: provider, + PublicationMode: publicationMode, + Phase: PhaseInitialized, + OutputDir: outputDir, + } +} + +func OrderedPhases() []Phase { + phases := make([]Phase, len(orderedPhases)) + copy(phases, orderedPhases) + return phases +} + +func (c Checkpoint) Validate() error { + if c.SchemaVersion != CheckpointSchemaVersion { + return fmt.Errorf("unsupported rotation checkpoint schema version %d", c.SchemaVersion) + } + + if !isSupportedProvider(c.Provider) { + return fmt.Errorf("unsupported rotation provider %q", c.Provider) + } + + if !isSupportedPublicationMode(c.PublicationMode) { + return fmt.Errorf("unsupported rotation publication mode %q", c.PublicationMode) + } + + if !isSupportedPhase(c.Phase) { + return fmt.Errorf("unsupported rotation phase %q", c.Phase) + } + + if strings.TrimSpace(c.OutputDir) == "" { + return fmt.Errorf("rotation output directory must not be empty") + } + + artifacts := make(map[string]ArtifactMetadata, len(c.Artifacts)) + for _, artifact := range c.Artifacts { + if err := validateArtifact(artifact); err != nil { + return err + } + if _, exists := artifacts[artifact.Name]; exists { + return fmt.Errorf("duplicate rotation artifact %q", artifact.Name) + } + artifacts[artifact.Name] = artifact + } + allowedArtifacts := make(map[string]struct{}, len(requiredArtifacts(c.Phase))) + for _, artifactName := range requiredArtifacts(c.Phase) { + allowedArtifacts[artifactName] = struct{}{} + } + for artifactName := range artifacts { + if _, allowed := allowedArtifacts[artifactName]; !allowed { + return fmt.Errorf("rotation artifact %q is ahead of checkpoint phase %q", artifactName, c.Phase) + } + } + + publications := make(map[Phase]PublicationConfirmation, len(c.Publications)) + for _, publication := range c.Publications { + if err := validatePublicationConfirmation(publication, artifacts); err != nil { + return err + } + if _, exists := publications[publication.Phase]; exists { + return fmt.Errorf("duplicate rotation publication confirmation for phase %q", publication.Phase) + } + if phasePosition(c.Phase) < phasePosition(publication.Phase) { + return fmt.Errorf("rotation publication confirmation for phase %q is ahead of checkpoint phase %q", publication.Phase, c.Phase) + } + publications[publication.Phase] = publication + } + + if phaseAtLeast(c.Phase, PhasePreflightComplete) { + if strings.TrimSpace(c.ClusterIdentity) == "" { + return fmt.Errorf("rotation cluster identity must be recorded after preflight") + } + if strings.TrimSpace(c.TargetIdentity) == "" { + return fmt.Errorf("rotation target identity must be recorded after preflight") + } + if c.PreRotationSignerBaseline == nil { + return fmt.Errorf("pre-rotation public signer baseline must be recorded after preflight") + } + if err := validatePublicSignerBaseline(*c.PreRotationSignerBaseline); err != nil { + return fmt.Errorf("invalid pre-rotation public signer baseline: %w", err) + } + if c.PreRotationSignerRef == nil { + return fmt.Errorf("pre-rotation signer object reference must be recorded after preflight") + } + if err := validateSignerObjectReference(*c.PreRotationSignerRef); err != nil { + return fmt.Errorf("invalid pre-rotation signer object reference: %w", err) + } + if c.RotationGuard == nil { + return fmt.Errorf("rotation guard reference must be recorded after preflight") + } + if err := validateRotationGuardReference(*c.RotationGuard); err != nil { + return fmt.Errorf("invalid rotation guard reference: %w", err) + } + expectedGuard, err := deriveRotationGuardReference(c.ClusterIdentity, c.Provider, c.TargetIdentity, *c.PreRotationSignerBaseline, *c.PreRotationSignerRef) + if err != nil { + return fmt.Errorf("derive rotation guard reference: %w", err) + } + if *c.RotationGuard != expectedGuard { + return fmt.Errorf("rotation guard reference does not match the recorded preflight evidence") + } + } else if c.ClusterIdentity != "" || c.TargetIdentity != "" || c.PreRotationSignerBaseline != nil || c.PreRotationSignerRef != nil || c.RotationGuard != nil { + return fmt.Errorf("rotation identities must not be recorded before preflight completes") + } + + if phaseAtLeast(c.Phase, PhaseNextPublicKeyRead) { + if c.ReplacementSigner == nil { + return fmt.Errorf("replacement signer evidence must be recorded after reading the replacement public key") + } + if err := validateReplacementSignerEvidence(*c.ReplacementSigner, c.PreRotationSignerBaseline); err != nil { + return fmt.Errorf("invalid replacement signer evidence: %w", err) + } + if c.PreRotationSignerRef != nil && c.ReplacementSigner.SecretRef.UID == c.PreRotationSignerRef.UID { + return fmt.Errorf("replacement signer Secret UID must differ from the pre-rotation signer Secret UID") + } + replacementArtifact, exists := artifacts[ArtifactReplacementPublicKey] + if exists && (replacementArtifact.SHA256 != c.ReplacementSigner.Entry.SHA256 || len(replacementArtifact.KeyIDs) != 1 || replacementArtifact.KeyIDs[0] != c.ReplacementSigner.Entry.KeyID) { + return fmt.Errorf("replacement signer evidence does not match artifact %q", ArtifactReplacementPublicKey) + } + } else if c.ReplacementSigner != nil { + return fmt.Errorf("replacement signer evidence is ahead of checkpoint phase %q", c.Phase) + } + + if phaseAtLeast(c.Phase, PhaseRebootIntentRecorded) { + if c.RebootIntent == nil { + return fmt.Errorf("rotation reboot intent must be recorded at phase %q", PhaseRebootIntentRecorded) + } + if err := validateRebootIntent(*c.RebootIntent); err != nil { + return fmt.Errorf("invalid rotation reboot intent: %w", err) + } + if replacement, exists := artifacts[ArtifactReplacementPublicKey]; exists { + expectedID, err := rebootIntentID(c.ClusterIdentity, replacement.KeyIDs[0]) + if err != nil { + return fmt.Errorf("derive rotation reboot intent ID: %w", err) + } + if c.RebootIntent.ID != expectedID { + return fmt.Errorf("rotation reboot intent ID does not match the cluster and replacement key") + } + } + } else if c.RebootIntent != nil { + return fmt.Errorf("rotation reboot intent is ahead of checkpoint phase %q", c.Phase) + } + + for _, requiredArtifact := range requiredArtifacts(c.Phase) { + if _, exists := artifacts[requiredArtifact]; !exists { + return fmt.Errorf("rotation phase %q requires artifact %q", c.Phase, requiredArtifact) + } + } + + for _, requiredPublication := range requiredPublications(c.Phase) { + if _, exists := publications[requiredPublication]; !exists { + return fmt.Errorf("rotation phase %q requires publication confirmation for phase %q", c.Phase, requiredPublication) + } + } + + return nil +} + +func isSupportedProvider(provider Provider) bool { + switch provider { + case ProviderAWS, ProviderAzure, ProviderGCP: + return true + default: + return false + } +} + +func isSupportedPublicationMode(mode PublicationMode) bool { + switch mode { + case PublicationModeDirect, PublicationModeManual: + return true + default: + return false + } +} + +func isSupportedPhase(phase Phase) bool { + for _, supportedPhase := range orderedPhases { + if phase == supportedPhase { + return true + } + } + return false +} + +func phasePosition(phase Phase) int { + for position, supportedPhase := range orderedPhases { + if phase == supportedPhase { + return position + } + } + return -1 +} + +func phaseAtLeast(current, required Phase) bool { + return phasePosition(current) >= phasePosition(required) +} + +func requiredArtifacts(phase Phase) []string { + required := []string{} + if phaseAtLeast(phase, PhaseCurrentJWKSRead) { + required = append(required, ArtifactCurrentJWKS) + } + if phaseAtLeast(phase, PhaseNextPublicKeyRead) { + required = append(required, ArtifactReplacementPublicKey) + } + if phaseAtLeast(phase, PhaseNewJWKSBuilt) { + required = append(required, ArtifactNewJWKS) + } + if phaseAtLeast(phase, PhaseCombinedJWKSBuilt) { + required = append(required, ArtifactCombinedJWKS) + } + return required +} + +func requiredPublications(phase Phase) []Phase { + required := []Phase{} + if phaseAtLeast(phase, PhaseCombinedJWKSPublished) { + required = append(required, PhaseCombinedJWKSPublished) + } + if phaseAtLeast(phase, PhaseNewOnlyJWKSPublished) { + required = append(required, PhaseNewOnlyJWKSPublished) + } + return required +} + +func validateArtifact(artifact ArtifactMetadata) error { + switch artifact.Name { + case ArtifactReplacementPublicKey, ArtifactCurrentJWKS, ArtifactNewJWKS, ArtifactCombinedJWKS: + default: + return fmt.Errorf("unsupported rotation artifact %q", artifact.Name) + } + + if err := validateSHA256(artifact.SHA256); err != nil { + return fmt.Errorf("rotation artifact %q: %w", artifact.Name, err) + } + if len(artifact.KeyIDs) == 0 { + return fmt.Errorf("rotation artifact %q must record at least one key ID", artifact.Name) + } + seenKeyIDs := make(map[string]struct{}, len(artifact.KeyIDs)) + for _, keyID := range artifact.KeyIDs { + if keyID == "" || strings.TrimSpace(keyID) != keyID { + return fmt.Errorf("rotation artifact %q contains an invalid key ID", artifact.Name) + } + if _, exists := seenKeyIDs[keyID]; exists { + return fmt.Errorf("rotation artifact %q contains duplicate key ID %q", artifact.Name, keyID) + } + seenKeyIDs[keyID] = struct{}{} + } + if (artifact.Name == ArtifactReplacementPublicKey || artifact.Name == ArtifactNewJWKS) && len(artifact.KeyIDs) != 1 { + return fmt.Errorf("rotation artifact %q must contain exactly one key ID", artifact.Name) + } + if artifact.Name == ArtifactCombinedJWKS && len(artifact.KeyIDs) < 2 { + return fmt.Errorf("rotation artifact %q must contain at least two key IDs", artifact.Name) + } + + return nil +} + +func validatePublicationConfirmation(publication PublicationConfirmation, artifacts map[string]ArtifactMetadata) error { + var expectedArtifact string + switch publication.Phase { + case PhaseCombinedJWKSPublished: + expectedArtifact = ArtifactCombinedJWKS + case PhaseNewOnlyJWKSPublished: + expectedArtifact = ArtifactNewJWKS + default: + return fmt.Errorf("unsupported rotation publication confirmation phase %q", publication.Phase) + } + + if publication.Artifact != expectedArtifact { + return fmt.Errorf("rotation publication confirmation for phase %q must reference artifact %q", publication.Phase, expectedArtifact) + } + if err := validateSHA256(publication.SHA256); err != nil { + return fmt.Errorf("rotation publication confirmation for phase %q: %w", publication.Phase, err) + } + + artifact, exists := artifacts[publication.Artifact] + if !exists { + return fmt.Errorf("rotation publication confirmation for phase %q references missing artifact %q", publication.Phase, publication.Artifact) + } + if publication.SHA256 != artifact.SHA256 { + return fmt.Errorf("rotation publication confirmation for phase %q does not match artifact %q digest", publication.Phase, publication.Artifact) + } + + return nil +} + +func validateSHA256(digest string) error { + if len(digest) != 64 { + return fmt.Errorf("SHA-256 digest must contain 64 hexadecimal characters") + } + if _, err := hex.DecodeString(digest); err != nil { + return fmt.Errorf("SHA-256 digest must contain only hexadecimal characters") + } + return nil +} + +func validateDerivedKeyID(keyID string) error { + if strings.TrimSpace(keyID) != keyID { + return fmt.Errorf("key ID must not contain surrounding whitespace") + } + decoded, err := base64.RawURLEncoding.Strict().DecodeString(keyID) + if err != nil || len(decoded) != 32 { + return fmt.Errorf("key ID must be an unpadded base64url-encoded SHA-256 digest") + } + return nil +} + +func validateSignerObjectReference(reference SignerObjectReference) error { + if err := validateOpaqueCheckpointValue("signer UID", reference.UID); err != nil { + return err + } + if err := validateOpaqueCheckpointValue("signer resource version", reference.ResourceVersion); err != nil { + return err + } + return nil +} + +func validatePublicSignerBaseline(baseline PublicSignerBaseline) error { + if err := validateOpaqueCheckpointValue("public signer ConfigMap UID", baseline.ConfigMapUID); err != nil { + return err + } + if err := validateOpaqueCheckpointValue("public signer ConfigMap resource version", baseline.ConfigMapResourceVersion); err != nil { + return err + } + if len(baseline.Entries) == 0 { + return fmt.Errorf("public signer baseline must contain at least one entry") + } + + names := make(map[string]struct{}, len(baseline.Entries)) + digests := make(map[string]struct{}, len(baseline.Entries)) + keyIDs := make(map[string]struct{}, len(baseline.Entries)) + previousName := "" + for _, entry := range baseline.Entries { + if err := validatePublicSignerBaselineEntry(entry); err != nil { + return err + } + if previousName != "" && entry.Name <= previousName { + return fmt.Errorf("public signer baseline entries must be sorted by unique name") + } + previousName = entry.Name + if _, exists := names[entry.Name]; exists { + return fmt.Errorf("public signer baseline contains duplicate entry name %q", entry.Name) + } + names[entry.Name] = struct{}{} + if _, exists := digests[entry.SHA256]; exists { + return fmt.Errorf("public signer baseline contains duplicate public value digest %q", entry.SHA256) + } + digests[entry.SHA256] = struct{}{} + if _, exists := keyIDs[entry.KeyID]; exists { + return fmt.Errorf("public signer baseline contains duplicate key ID %q", entry.KeyID) + } + keyIDs[entry.KeyID] = struct{}{} + } + return nil +} + +func validatePublicSignerBaselineEntry(entry PublicSignerBaselineEntry) error { + if !publicSignerEntryNamePattern.MatchString(entry.Name) { + return fmt.Errorf("public signer entry name %q is not supported", entry.Name) + } + if err := validateSHA256(entry.SHA256); err != nil { + return fmt.Errorf("public signer entry %q: %w", entry.Name, err) + } + if err := validateDerivedKeyID(entry.KeyID); err != nil { + return fmt.Errorf("public signer entry %q has invalid key ID: %w", entry.Name, err) + } + return nil +} + +func validateReplacementSignerEvidence(evidence ReplacementSignerEvidence, baseline *PublicSignerBaseline) error { + if err := validatePublicSignerBaselineEntry(evidence.Entry); err != nil { + return err + } + if err := validateSignerObjectReference(evidence.SecretRef); err != nil { + return err + } + if baseline == nil { + return fmt.Errorf("replacement signer evidence requires a pre-rotation public signer baseline") + } + for _, entry := range baseline.Entries { + switch { + case entry.Name == evidence.Entry.Name: + return fmt.Errorf("replacement signer entry name %q already exists in the pre-rotation baseline", evidence.Entry.Name) + case entry.SHA256 == evidence.Entry.SHA256: + return fmt.Errorf("replacement signer repeats a pre-rotation public value") + case entry.KeyID == evidence.Entry.KeyID: + return fmt.Errorf("replacement signer key ID %q already exists in the pre-rotation baseline", evidence.Entry.KeyID) + } + } + return nil +} + +func validateRebootIntent(intent RebootIntent) error { + if err := validateOpaqueCheckpointValue("reboot intent ID", intent.ID); err != nil { + return err + } + if len(intent.Targets) == 0 { + return fmt.Errorf("reboot intent must contain at least one target") + } + + targets := make(map[string]struct{}, len(intent.Targets)) + for _, target := range intent.Targets { + if err := validateOpaqueCheckpointValue("reboot target", target); err != nil { + return err + } + if _, exists := targets[target]; exists { + return fmt.Errorf("reboot intent contains duplicate target %q", target) + } + targets[target] = struct{}{} + } + + if len(intent.Baselines) == 0 { + return fmt.Errorf("reboot intent must contain at least one node baseline") + } + baselineCounts := make(map[string]int, len(targets)) + nodes := make(map[string]struct{}, len(intent.Baselines)) + for _, baseline := range intent.Baselines { + if err := validateOpaqueCheckpointValue("reboot baseline target", baseline.Target); err != nil { + return err + } + if _, exists := targets[baseline.Target]; !exists { + return fmt.Errorf("reboot baseline for node %q references unknown target %q", baseline.Node, baseline.Target) + } + if err := validateOpaqueCheckpointValue("reboot baseline node", baseline.Node); err != nil { + return err + } + if _, exists := nodes[baseline.Node]; exists { + return fmt.Errorf("reboot intent contains duplicate node baseline %q", baseline.Node) + } + nodes[baseline.Node] = struct{}{} + if err := validateOpaqueCheckpointValue("reboot baseline boot ID", baseline.BootID); err != nil { + return err + } + baselineCounts[baseline.Target]++ + } + for _, target := range intent.Targets { + if baselineCounts[target] == 0 { + return fmt.Errorf("reboot target %q has no node baseline", target) + } + } + return nil +} + +func validateOpaqueCheckpointValue(name, value string) error { + if value == "" { + return fmt.Errorf("%s must not be empty", name) + } + if strings.TrimSpace(value) != value { + return fmt.Errorf("%s must not contain surrounding whitespace", name) + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("%s must not contain control characters", name) + } + } + return nil +} diff --git a/pkg/cmd/provisioning/rotation/contract_test.go b/pkg/cmd/provisioning/rotation/contract_test.go new file mode 100644 index 0000000000..9001310bcd --- /dev/null +++ b/pkg/cmd/provisioning/rotation/contract_test.go @@ -0,0 +1,929 @@ +package rotation + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +const ( + testDigest = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testReplacementDigest = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + testOldKeyID = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + testOtherKeyID = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE" +) + +func TestCheckpointValidate(t *testing.T) { + tests := []struct { + name string + checkpoint Checkpoint + wantError string + }{ + { + name: "valid direct checkpoint", + checkpoint: NewCheckpoint(ProviderAWS, PublicationModeDirect, "rotation-output"), + }, + { + name: "valid manual checkpoint with public artifacts", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAzure, + PublicationMode: PublicationModeManual, + Phase: PhaseCombinedJWKSBuilt, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "azure://account/container/openid/v1/jwks", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + RotationGuard: testRotationGuard(ProviderAzure, "cluster-123", "azure://account/container/openid/v1/jwks", testSignerBaseline(), testSignerObjectReference()), + ReplacementSigner: testReplacementSignerEvidence(), + Artifacts: []ArtifactMetadata{ + {Name: ArtifactReplacementPublicKey, SHA256: testReplacementDigest, KeyIDs: []string{testOtherKeyID}}, + {Name: ArtifactCurrentJWKS, SHA256: testDigest, KeyIDs: []string{testOldKeyID}}, + {Name: ArtifactNewJWKS, SHA256: testDigest, KeyIDs: []string{testOtherKeyID}}, + {Name: ArtifactCombinedJWKS, SHA256: testDigest, KeyIDs: []string{testOldKeyID, testOtherKeyID}}, + }, + }, + }, + { + name: "unsupported schema", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion + 1, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseInitialized, + OutputDir: "rotation-output", + }, + wantError: "unsupported rotation checkpoint schema version", + }, + { + name: "unsupported provider", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: "openstack", + PublicationMode: PublicationModeDirect, + Phase: PhaseInitialized, + OutputDir: "rotation-output", + }, + wantError: "unsupported rotation provider", + }, + { + name: "unsupported publication mode", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: "automatic", + Phase: PhaseInitialized, + OutputDir: "rotation-output", + }, + wantError: "unsupported rotation publication mode", + }, + { + name: "unsupported phase", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: "final-jwks-published-early", + OutputDir: "rotation-output", + }, + wantError: "unsupported rotation phase", + }, + { + name: "missing output directory", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseInitialized, + }, + wantError: "rotation output directory must not be empty", + }, + { + name: "private key artifact", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseNextPublicKeyRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{{Name: "serviceaccount-signer.private"}}, + }, + wantError: "unsupported rotation artifact", + }, + { + name: "private key extension", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseNextPublicKeyRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{{Name: "signer.key"}}, + }, + wantError: "unsupported rotation artifact", + }, + { + name: "artifact outside output directory", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseCurrentJWKSRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{{Name: "../jwks.current.json"}}, + }, + wantError: "unsupported rotation artifact", + }, + { + name: "windows path outside output directory", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseCurrentJWKSRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{{Name: `..\\jwks.current.json`}}, + }, + wantError: "unsupported rotation artifact", + }, + { + name: "parent directory artifact", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseCurrentJWKSRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{{Name: ".."}}, + }, + wantError: "unsupported rotation artifact", + }, + { + name: "unrecognized public artifact", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseCurrentJWKSRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{{Name: "signer.pem", SHA256: testDigest}}, + }, + wantError: "unsupported rotation artifact", + }, + { + name: "invalid artifact digest", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseCurrentJWKSRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{{Name: ArtifactCurrentJWKS, SHA256: "not-a-digest"}}, + }, + wantError: "SHA-256 digest", + }, + { + name: "duplicate artifact", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseCurrentJWKSRead, + OutputDir: "rotation-output", + TargetIdentity: "aws://issuer", + Artifacts: []ArtifactMetadata{ + {Name: ArtifactCurrentJWKS, SHA256: testDigest, KeyIDs: []string{"old"}}, + {Name: ArtifactCurrentJWKS, SHA256: testDigest, KeyIDs: []string{"old"}}, + }, + }, + wantError: "duplicate rotation artifact", + }, + { + name: "missing cluster identity after preflight", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + TargetIdentity: "gcp://bucket/keys.json", + }, + wantError: "cluster identity must be recorded", + }, + { + name: "missing target after preflight", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + }, + wantError: "target identity must be recorded", + }, + { + name: "missing pre-rotation public signer baseline after preflight", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "gcp://bucket/keys.json", + }, + wantError: "public signer baseline must be recorded", + }, + { + name: "pre-rotation public signer baseline with invalid entry name", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "gcp://bucket/keys.json", + PreRotationSignerBaseline: testSignerBaselineWithEntry(PublicSignerBaselineEntry{Name: " service-account-001.pub", SHA256: testDigest, KeyID: testOldKeyID}), + PreRotationSignerRef: testSignerObjectReference(), + }, + wantError: "entry name", + }, + { + name: "malformed pre-rotation signer key identity", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "gcp://bucket/keys.json", + PreRotationSignerBaseline: testSignerBaselineWithEntry(PublicSignerBaselineEntry{Name: "service-account-001.pub", SHA256: testDigest, KeyID: "not-a-derived-key-id"}), + PreRotationSignerRef: testSignerObjectReference(), + }, + wantError: "base64url-encoded SHA-256", + }, + { + name: "identity evidence before preflight", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhaseInitialized, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + }, + wantError: "must not be recorded before preflight", + }, + { + name: "missing pre-rotation signer reference after preflight", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "gcp://bucket/keys.json", + PreRotationSignerBaseline: testSignerBaseline(), + }, + wantError: "signer object reference must be recorded", + }, + { + name: "invalid pre-rotation signer UID", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "gcp://bucket/keys.json", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: &SignerObjectReference{ + UID: " signer-uid", + ResourceVersion: "12345", + }, + }, + wantError: "signer UID must not contain surrounding whitespace", + }, + { + name: "invalid pre-rotation signer resource version", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderGCP, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "gcp://bucket/keys.json", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: &SignerObjectReference{ + UID: "signer-uid", + }, + }, + wantError: "signer resource version must not be empty", + }, + { + name: "missing rotation guard after preflight", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "aws://issuer", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + }, + wantError: "rotation guard reference must be recorded", + }, + { + name: "rotation guard does not match preflight evidence", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhasePreflightComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "aws://issuer", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + RotationGuard: &RotationGuardReference{ + ScopeID: strings.Repeat("1", 64), + OperationID: strings.Repeat("2", 64), + }, + }, + wantError: "does not match the recorded preflight evidence", + }, + { + name: "complete phase missing required artifacts", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "aws://issuer", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + RotationGuard: testRotationGuard(ProviderAWS, "cluster-123", "aws://issuer", testSignerBaseline(), testSignerObjectReference()), + ReplacementSigner: testReplacementSignerEvidence(), + RebootIntent: testNodeRebootIntent(), + }, + wantError: "requires artifact", + }, + { + name: "combined publication missing confirmation", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeManual, + Phase: PhaseCombinedJWKSPublished, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "aws://issuer", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + RotationGuard: testRotationGuard(ProviderAWS, "cluster-123", "aws://issuer", testSignerBaseline(), testSignerObjectReference()), + ReplacementSigner: testReplacementSignerEvidence(), + Artifacts: allTestArtifacts(), + }, + wantError: "requires publication confirmation", + }, + { + name: "publication confirmation digest mismatch", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeManual, + Phase: PhaseCombinedJWKSPublished, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "aws://issuer", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + RotationGuard: testRotationGuard(ProviderAWS, "cluster-123", "aws://issuer", testSignerBaseline(), testSignerObjectReference()), + ReplacementSigner: testReplacementSignerEvidence(), + Artifacts: allTestArtifacts(), + Publications: []PublicationConfirmation{{ + Phase: PhaseCombinedJWKSPublished, + Artifact: ArtifactCombinedJWKS, + SHA256: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }}, + }, + wantError: "does not match", + }, + { + name: "publication confirmation ahead of checkpoint", + checkpoint: Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeManual, + Phase: PhaseCombinedJWKSBuilt, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "aws://issuer", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + RotationGuard: testRotationGuard(ProviderAWS, "cluster-123", "aws://issuer", testSignerBaseline(), testSignerObjectReference()), + ReplacementSigner: testReplacementSignerEvidence(), + Artifacts: allTestArtifacts(), + Publications: []PublicationConfirmation{{ + Phase: PhaseCombinedJWKSPublished, + Artifact: ArtifactCombinedJWKS, + SHA256: testDigest, + }}, + }, + wantError: "ahead of checkpoint phase", + }, + { + name: "valid complete checkpoint", + checkpoint: completeTestCheckpoint(), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.checkpoint.Validate() + if test.wantError == "" { + if err != nil { + t.Fatalf("Validate() returned unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("Validate() error = %v, want an error containing %q", err, test.wantError) + } + }) + } +} + +func TestCheckpointValidateRebootIntent(t *testing.T) { + base := completeTestCheckpoint() + base.Phase = PhaseRebootIntentRecorded + base.Publications = base.Publications[:1] + + tests := []struct { + name string + mutate func(*Checkpoint) + wantError string + }{ + {name: "valid reboot intent"}, + { + name: "missing reboot intent", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent = nil + }, + wantError: "reboot intent must be recorded", + }, + { + name: "reboot intent before its phase", + mutate: func(checkpoint *Checkpoint) { + checkpoint.Phase = PhaseSignerRolloutStable + }, + wantError: "reboot intent is ahead", + }, + { + name: "missing reboot intent ID", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.ID = "" + }, + wantError: "reboot intent ID must not be empty", + }, + { + name: "reboot intent ID not bound to checkpoint", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.ID = "different-valid-looking-id" + }, + wantError: "does not match the cluster and replacement key", + }, + { + name: "cluster identity changed without a new intent ID", + mutate: func(checkpoint *Checkpoint) { + checkpoint.ClusterIdentity = "other-cluster" + setTestRotationGuard(checkpoint) + }, + wantError: "does not match the cluster and replacement key", + }, + { + name: "missing reboot targets", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.Targets = nil + }, + wantError: "at least one target", + }, + { + name: "duplicate reboot target", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.Targets = []string{"worker", "worker"} + }, + wantError: "duplicate target", + }, + { + name: "missing reboot baselines", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.Baselines = nil + }, + wantError: "at least one node baseline", + }, + { + name: "unknown baseline target", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.Baselines[0].Target = "infra" + }, + wantError: "references unknown target", + }, + { + name: "duplicate baseline node", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.Baselines[1].Node = checkpoint.RebootIntent.Baselines[0].Node + }, + wantError: "duplicate node baseline", + }, + { + name: "missing baseline boot ID", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.Baselines[0].BootID = "" + }, + wantError: "boot ID must not be empty", + }, + { + name: "target without baseline", + mutate: func(checkpoint *Checkpoint) { + checkpoint.RebootIntent.Baselines = checkpoint.RebootIntent.Baselines[:1] + }, + wantError: "has no node baseline", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + checkpoint := base + checkpoint.RebootIntent = testNodeRebootIntent() + if test.mutate != nil { + test.mutate(&checkpoint) + } + err := checkpoint.Validate() + if test.wantError == "" { + if err != nil { + t.Fatalf("Validate() returned unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("Validate() error = %v, want an error containing %q", err, test.wantError) + } + }) + } +} + +func TestCheckpointValidateSignerEvidence(t *testing.T) { + secondEntry := PublicSignerBaselineEntry{ + Name: "service-account-002.pub", + SHA256: strings.Repeat("c", 64), + KeyID: testOtherKeyID, + } + tests := []struct { + name string + mutate func(*Checkpoint) + wantError string + }{ + {name: "valid signer evidence"}, + { + name: "baseline entries are not canonical", + mutate: func(checkpoint *Checkpoint) { + checkpoint.PreRotationSignerBaseline.Entries = []PublicSignerBaselineEntry{secondEntry, checkpoint.PreRotationSignerBaseline.Entries[0]} + }, + wantError: "must be sorted", + }, + { + name: "baseline repeats exact public value", + mutate: func(checkpoint *Checkpoint) { + duplicate := secondEntry + duplicate.SHA256 = checkpoint.PreRotationSignerBaseline.Entries[0].SHA256 + checkpoint.PreRotationSignerBaseline.Entries = append(checkpoint.PreRotationSignerBaseline.Entries, duplicate) + }, + wantError: "duplicate public value digest", + }, + { + name: "baseline repeats semantic key", + mutate: func(checkpoint *Checkpoint) { + duplicate := secondEntry + duplicate.KeyID = checkpoint.PreRotationSignerBaseline.Entries[0].KeyID + checkpoint.PreRotationSignerBaseline.Entries = append(checkpoint.PreRotationSignerBaseline.Entries, duplicate) + }, + wantError: "duplicate key ID", + }, + { + name: "missing replacement evidence", + mutate: func(checkpoint *Checkpoint) { + checkpoint.ReplacementSigner = nil + }, + wantError: "replacement signer evidence must be recorded", + }, + { + name: "replacement repeats baseline name", + mutate: func(checkpoint *Checkpoint) { + checkpoint.ReplacementSigner.Entry.Name = checkpoint.PreRotationSignerBaseline.Entries[0].Name + }, + wantError: "already exists", + }, + { + name: "replacement repeats baseline key ID", + mutate: func(checkpoint *Checkpoint) { + checkpoint.ReplacementSigner.Entry.KeyID = checkpoint.PreRotationSignerBaseline.Entries[0].KeyID + }, + wantError: "already exists", + }, + { + name: "replacement artifact binding differs", + mutate: func(checkpoint *Checkpoint) { + checkpoint.ReplacementSigner.Entry.SHA256 = strings.Repeat("d", 64) + }, + wantError: "does not match artifact", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + checkpoint := cloneCheckpoint(completeTestCheckpoint()) + if test.mutate != nil { + test.mutate(&checkpoint) + } + err := checkpoint.Validate() + if test.wantError == "" { + if err != nil { + t.Fatalf("Validate() returned unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("Validate() error = %v, want %q", err, test.wantError) + } + }) + } +} + +func TestCloneCheckpointDoesNotExposeSignerEvidenceAliases(t *testing.T) { + original := completeTestCheckpoint() + clone := cloneCheckpoint(original) + clone.PreRotationSignerBaseline.Entries[0].SHA256 = strings.Repeat("c", 64) + clone.PreRotationSignerRef.ResourceVersion = "changed" + clone.RotationGuard.OperationID = strings.Repeat("d", 64) + clone.ReplacementSigner.SecretRef.ResourceVersion = "changed" + + if reflect.DeepEqual(original.PreRotationSignerBaseline, clone.PreRotationSignerBaseline) || original.PreRotationSignerRef.ResourceVersion == "changed" || original.RotationGuard.OperationID == strings.Repeat("d", 64) || original.ReplacementSigner.SecretRef.ResourceVersion == "changed" { + t.Fatal("cloneCheckpoint() exposed mutable signer evidence aliases") + } +} + +func TestOrderedPhasesPreserveSafeRotationOrder(t *testing.T) { + phases := OrderedPhases() + expected := []Phase{ + PhaseInitialized, + PhasePreflightComplete, + PhaseGuardAcquired, + PhaseCurrentJWKSRead, + PhaseNextKeyRequested, + PhaseNextPublicKeyRead, + PhaseNewJWKSBuilt, + PhaseCombinedJWKSBuilt, + PhaseCombinedJWKSPublished, + PhaseSignerRolloutStable, + PhaseRebootIntentRecorded, + PhaseNodesRebooted, + PhasePostRebootStable, + PhaseNewOnlyJWKSPublished, + PhaseGuardReleaseRecorded, + PhaseComplete, + } + if !reflect.DeepEqual(phases, expected) { + t.Fatalf("OrderedPhases() = %v, want %v", phases, expected) + } + + position := make(map[Phase]int, len(phases)) + for index, phase := range phases { + position[phase] = index + } + + assertBefore := func(earlier, later Phase) { + t.Helper() + earlierPosition, earlierExists := position[earlier] + laterPosition, laterExists := position[later] + if !earlierExists || !laterExists { + t.Fatalf("required phases %q and %q must both be present", earlier, later) + } + if earlierPosition >= laterPosition { + t.Fatalf("phase %q must precede %q", earlier, later) + } + } + + assertBefore(PhasePreflightComplete, PhaseGuardAcquired) + assertBefore(PhaseGuardAcquired, PhaseCurrentJWKSRead) + assertBefore(PhaseCurrentJWKSRead, PhaseNextKeyRequested) + assertBefore(PhaseCombinedJWKSPublished, PhaseSignerRolloutStable) + assertBefore(PhaseSignerRolloutStable, PhaseRebootIntentRecorded) + assertBefore(PhaseRebootIntentRecorded, PhaseNodesRebooted) + assertBefore(PhaseNodesRebooted, PhasePostRebootStable) + assertBefore(PhasePostRebootStable, PhaseNewOnlyJWKSPublished) + assertBefore(PhaseNewOnlyJWKSPublished, PhaseGuardReleaseRecorded) + assertBefore(PhaseGuardReleaseRecorded, PhaseComplete) +} + +func allTestArtifacts() []ArtifactMetadata { + return []ArtifactMetadata{ + {Name: ArtifactCurrentJWKS, SHA256: testDigest, KeyIDs: []string{testOldKeyID}}, + {Name: ArtifactReplacementPublicKey, SHA256: testReplacementDigest, KeyIDs: []string{testOtherKeyID}}, + {Name: ArtifactNewJWKS, SHA256: testDigest, KeyIDs: []string{testOtherKeyID}}, + {Name: ArtifactCombinedJWKS, SHA256: testDigest, KeyIDs: []string{testOldKeyID, testOtherKeyID}}, + } +} + +func completeTestCheckpoint() Checkpoint { + checkpoint := Checkpoint{ + SchemaVersion: CheckpointSchemaVersion, + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + Phase: PhaseComplete, + OutputDir: "rotation-output", + ClusterIdentity: "cluster-123", + TargetIdentity: "aws://issuer", + PreRotationSignerBaseline: testSignerBaseline(), + PreRotationSignerRef: testSignerObjectReference(), + ReplacementSigner: testReplacementSignerEvidence(), + RebootIntent: testNodeRebootIntent(), + Artifacts: allTestArtifacts(), + Publications: []PublicationConfirmation{ + {Phase: PhaseCombinedJWKSPublished, Artifact: ArtifactCombinedJWKS, SHA256: testDigest}, + {Phase: PhaseNewOnlyJWKSPublished, Artifact: ArtifactNewJWKS, SHA256: testDigest}, + }, + } + checkpoint.RotationGuard = testRotationGuard(checkpoint.Provider, checkpoint.ClusterIdentity, checkpoint.TargetIdentity, checkpoint.PreRotationSignerBaseline, checkpoint.PreRotationSignerRef) + return checkpoint +} + +func testRotationGuard(provider Provider, clusterIdentity, targetIdentity string, baseline *PublicSignerBaseline, signerRef *SignerObjectReference) *RotationGuardReference { + reference, err := deriveRotationGuardReference(clusterIdentity, provider, targetIdentity, *baseline, *signerRef) + if err != nil { + panic(fmt.Sprintf("derive test rotation guard: %v", err)) + } + return &reference +} + +func setTestRotationGuard(checkpoint *Checkpoint) { + checkpoint.RotationGuard = testRotationGuard(checkpoint.Provider, checkpoint.ClusterIdentity, checkpoint.TargetIdentity, checkpoint.PreRotationSignerBaseline, checkpoint.PreRotationSignerRef) +} + +func testSignerBaseline() *PublicSignerBaseline { + return testSignerBaselineForKeyID(testOldKeyID) +} + +func testSignerBaselineForKeyID(keyID string) *PublicSignerBaseline { + return testSignerBaselineWithEntry(PublicSignerBaselineEntry{ + Name: "service-account-001.pub", + SHA256: testDigest, + KeyID: keyID, + }) +} + +func testSignerBaselineForKeyIDs(keyIDs ...string) *PublicSignerBaseline { + entries := make([]PublicSignerBaselineEntry, len(keyIDs)) + for index, keyID := range keyIDs { + entries[index] = PublicSignerBaselineEntry{ + Name: fmt.Sprintf("service-account-%03d.pub", index+1), + SHA256: fmt.Sprintf("%064x", index+1), + KeyID: keyID, + } + } + return &PublicSignerBaseline{ + ConfigMapUID: "aaaaaaaa-2222-3333-4444-555555555555", + ConfigMapResourceVersion: "67890", + Entries: entries, + } +} + +func testReplacementEvidenceForMetadata(metadata ArtifactMetadata) *ReplacementSignerEvidence { + return &ReplacementSignerEvidence{ + Entry: PublicSignerBaselineEntry{ + Name: "service-account-002.pub", + SHA256: metadata.SHA256, + KeyID: metadata.KeyIDs[0], + }, + SecretRef: SignerObjectReference{ + UID: "test-replacement-signer-uid", + ResourceVersion: "2000", + }, + } +} + +func testSignerBaselineWithEntry(entry PublicSignerBaselineEntry) *PublicSignerBaseline { + return &PublicSignerBaseline{ + ConfigMapUID: "aaaaaaaa-2222-3333-4444-555555555555", + ConfigMapResourceVersion: "67890", + Entries: []PublicSignerBaselineEntry{entry}, + } +} + +func testReplacementSignerEvidence() *ReplacementSignerEvidence { + return &ReplacementSignerEvidence{ + Entry: PublicSignerBaselineEntry{ + Name: "service-account-002.pub", + SHA256: testReplacementDigest, + KeyID: testOtherKeyID, + }, + SecretRef: SignerObjectReference{ + UID: "99999999-2222-3333-4444-555555555555", + ResourceVersion: "67891", + }, + } +} + +func testSignerBaselineForPublicKeys(t *testing.T, publicKeys ...[]byte) *PublicSignerBaseline { + t.Helper() + signers := make([]PublicSignerObservation, len(publicKeys)) + for index, publicKey := range publicKeys { + signers[index] = PublicSignerObservation{ + Name: fmt.Sprintf("service-account-%03d.pub", index+1), + PublicKeyPEM: append([]byte(nil), publicKey...), + } + } + state, err := normalizePublicSignerBundle(PublicSignerBundleObservation{ + ConfigMapUID: "test-signer-configmap-uid", + ConfigMapResourceVersion: "1000", + Signers: signers, + }) + if err != nil { + t.Fatalf("normalize test signer baseline: %v", err) + } + baseline := clonePublicSignerBaseline(state.Baseline) + return &baseline +} + +func testReplacementEvidenceForPublicKey(t *testing.T, publicKey []byte, entryIndex int) *ReplacementSignerEvidence { + t.Helper() + state, err := normalizePublicSignerBundle(PublicSignerBundleObservation{ + ConfigMapUID: "test-signer-configmap-uid", + ConfigMapResourceVersion: "1001", + Signers: []PublicSignerObservation{{ + Name: fmt.Sprintf("service-account-%03d.pub", entryIndex), + PublicKeyPEM: append([]byte(nil), publicKey...), + }}, + }) + if err != nil { + t.Fatalf("normalize test replacement signer: %v", err) + } + return &ReplacementSignerEvidence{ + Entry: state.Baseline.Entries[0], + SecretRef: SignerObjectReference{ + UID: "test-replacement-signer-uid", + ResourceVersion: "2000", + }, + } +} + +func testSignerObjectReference() *SignerObjectReference { + return &SignerObjectReference{ + UID: "11111111-2222-3333-4444-555555555555", + ResourceVersion: "12345", + } +} + +func testNodeRebootIntent() *RebootIntent { + targets := []string{"master", "worker"} + intentID, err := rebootIntentID("cluster-123", testOtherKeyID) + if err != nil { + panic(err) + } + return &RebootIntent{ + ID: intentID, + Targets: targets, + Baselines: []NodeRebootBaseline{ + {Target: "master", Node: "master-0", BootID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"}, + {Target: "worker", Node: "worker-0", BootID: "ffffffff-1111-2222-3333-444444444444"}, + }, + } +} + +func TestOrderedPhasesReturnsCopy(t *testing.T) { + phases := OrderedPhases() + phases[0] = PhaseComplete + + if OrderedPhases()[0] != PhaseInitialized { + t.Fatal("OrderedPhases() exposed mutable contract state") + } +} diff --git a/pkg/cmd/provisioning/rotation/current_jwks_binding_test.go b/pkg/cmd/provisioning/rotation/current_jwks_binding_test.go new file mode 100644 index 0000000000..15a119baf0 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/current_jwks_binding_test.go @@ -0,0 +1,116 @@ +package rotation + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidateArtifactRelationshipsBindsCurrentJWKSToPreRotationSignerBaseline(t *testing.T) { + expectedPublicKey := testPublicKeyPEM(t) + secondPublicKey := testPublicKeyPEM(t) + baseline := testSignerBaselineForPublicKeys(t, expectedPublicKey, secondPublicKey) + matchingCurrent := encodedJWKSForTest(t, expectedPublicKey) + matchingCombined, err := PrepareJWKSArtifacts(matchingCurrent, secondPublicKey) + if err != nil { + t.Fatalf("prepare matching multi-key JWKS: %v", err) + } + unrelatedCurrent := encodedJWKSForTest(t, testPublicKeyPEM(t)) + + tests := []struct { + name string + checkpoint Checkpoint + current []byte + wantError string + }{ + { + name: "matching current key", + checkpoint: Checkpoint{PreRotationSignerBaseline: baseline}, + current: matchingCurrent, + }, + { + name: "every current key belongs to baseline", + checkpoint: Checkpoint{PreRotationSignerBaseline: baseline}, + current: matchingCombined.Combined.Data, + }, + { + name: "unrelated current key", + checkpoint: Checkpoint{PreRotationSignerBaseline: baseline}, + current: unrelatedCurrent, + wantError: "is not present in the pre-rotation public signer baseline", + }, + { + name: "standalone artifact without preflight identity", + checkpoint: Checkpoint{}, + current: unrelatedCurrent, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateArtifactRelationships(test.checkpoint, map[string][]byte{ + ArtifactCurrentJWKS: test.current, + }) + if test.wantError == "" { + if err != nil { + t.Fatalf("validateArtifactRelationships() returned unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("validateArtifactRelationships() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestOrchestratorRejectsCurrentJWKSOutsidePreRotationBaselineBeforeReplacement(t *testing.T) { + for _, mode := range []PublicationMode{PublicationModeDirect, PublicationModeManual} { + t.Run(string(mode), func(t *testing.T) { + initial := safetySigner(testPublicKeyPEM(t), "old-uid", "10") + replacement := safetySigner(testPublicKeyPEM(t), "new-uid", "11") + cluster := newSafetyCluster(initial, replacement) + currentJWKS := encodedJWKSForTest(t, testPublicKeyPEM(t)) + orchestrator := Orchestrator{ + Cluster: cluster, + Target: safetyTarget("issuer-target"), + } + if mode == PublicationModeDirect { + orchestrator.Publisher = newSafetyPublisher(currentJWKS) + } + + outputDir := filepath.Join(t.TempDir(), "rotation") + options := RunOptions{ + Provider: ProviderAWS, + PublicationMode: mode, + OutputDir: outputDir, + } + if mode == PublicationModeManual { + result, err := orchestrator.Run(context.Background(), options) + var pause *PauseError + if !errors.As(err, &pause) || pause.Reason != PauseForCurrentJWKS || result.Phase != PhaseGuardAcquired { + t.Fatalf("initial manual Run() result = %#v, error = %v; want current-JWKS pause after preflight", result, err) + } + options.Resume = true + options.Manual.CurrentJWKS = currentJWKS + } + + result, err := orchestrator.Run(context.Background(), options) + if err == nil || !strings.Contains(err.Error(), "is not present in the pre-rotation public signer baseline") { + t.Fatalf("Run() error = %v, want current-JWKS identity mismatch", err) + } + if result.Phase != PhaseGuardAcquired || result.Complete { + t.Fatalf("Run() result = %#v, want failure at guard-acquired", result) + } + if cluster.replacementRequests != 0 { + t.Fatalf("replacement requests = %d, want zero", cluster.replacementRequests) + } + if _, err := os.Stat(filepath.Join(outputDir, ArtifactCurrentJWKS)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("mismatched current JWKS was persisted, stat error = %v", err) + } + }) + } +} diff --git a/pkg/cmd/provisioning/rotation/current_publication_recheck_test.go b/pkg/cmd/provisioning/rotation/current_publication_recheck_test.go new file mode 100644 index 0000000000..b014b1eaff --- /dev/null +++ b/pkg/cmd/provisioning/rotation/current_publication_recheck_test.go @@ -0,0 +1,117 @@ +package rotation + +import ( + "context" + "errors" + "strings" + "testing" +) + +func TestOrchestratorRechecksCurrentJWKSBeforeReplacementMutation(t *testing.T) { + tests := []struct { + name string + readError error + wantConflict bool + }{ + {name: "provider drift", wantConflict: true}, + {name: "provider read failure", readError: errors.New("current JWKS read unavailable")}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + publisher := ¤tRecheckPublisher{ + fakeConditionalJWKSBackend: harness.publisher, + failReadNumber: 2, + readError: test.readError, + } + if test.wantConflict { + publisher.driftedJWKS = encodedJWKSForTest(t, testPublicKeyPEM(t)) + } + harness.orchestrator.Publisher = publisher + + result, err := harness.orchestrator.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: t.TempDir(), + }) + if test.wantConflict { + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() error = %v, want ConflictError", err) + } + } else if err == nil || !strings.Contains(err.Error(), test.readError.Error()) { + t.Fatalf("Run() error = %v, want provider read failure", err) + } + if result.Phase != PhaseCurrentJWKSRead || result.Complete { + t.Fatalf("Run() result = %#v, want failure at current-jwks-read", result) + } + if harness.cluster.replacementRequests != 0 { + t.Fatalf("replacement requests = %d, want zero", harness.cluster.replacementRequests) + } + if got := countRotationEvent(harness.events, "cluster.observe-public-signer-bundle"); got != 3 { + t.Fatalf("public signer bundle observations = %d, want preflight, post-acquisition, and one convergence observation", got) + } + if got := countRotationEvent(harness.events, "cluster.request-replacement"); got != 0 { + t.Fatalf("replacement request events = %d, want zero", got) + } + }) + } +} + +func TestOrchestratorDoesNotRequireCurrentRecheckWhenSignerAlreadyChanged(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + harness.cluster.applyReplacementAt = 2 + publisher := ¤tRecheckPublisher{ + fakeConditionalJWKSBackend: harness.publisher, + failReadNumber: 2, + readError: errors.New("publication predecessor read unavailable"), + } + harness.orchestrator.Publisher = publisher + + result, err := harness.orchestrator.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: t.TempDir(), + }) + if err == nil || !strings.Contains(err.Error(), "before conditional publication") { + t.Fatalf("Run() error = %v, want failure at later combined publication", err) + } + if result.Phase != PhaseCombinedJWKSBuilt || result.Complete { + t.Fatalf("Run() result = %#v, want progress through replacement capture without a new mutation", result) + } + if harness.cluster.replacementRequests != 0 { + t.Fatalf("replacement requests = %d, want zero", harness.cluster.replacementRequests) + } +} + +type currentRecheckPublisher struct { + *fakeConditionalJWKSBackend + reads int + failReadNumber int + driftedJWKS []byte + readError error +} + +func (publisher *currentRecheckPublisher) ReadJWKS(ctx context.Context, target string) (VersionedJWKS, error) { + publisher.reads++ + if publisher.reads == publisher.failReadNumber { + if publisher.readError != nil { + *publisher.events = append(*publisher.events, "publisher.read:error") + return VersionedJWKS{}, publisher.readError + } + publisher.data = append([]byte(nil), publisher.driftedJWKS...) + publisher.revision++ + } + return publisher.fakeConditionalJWKSBackend.ReadJWKS(ctx, target) +} + +func countRotationEvent(events []string, want string) int { + count := 0 + for _, event := range events { + if event == want { + count++ + } + } + return count +} diff --git a/pkg/cmd/provisioning/rotation/jwks_artifacts.go b/pkg/cmd/provisioning/rotation/jwks_artifacts.go new file mode 100644 index 0000000000..5a5bef93c9 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/jwks_artifacts.go @@ -0,0 +1,50 @@ +package rotation + +import ( + "fmt" + + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" +) + +// PreparedJWKSArtifacts contains the exact validated provider state plus the +// deterministic replacement-only and overlapping key sets needed by rotation. +type PreparedJWKSArtifacts struct { + Current jwkutil.Artifact + New jwkutil.Artifact + Combined jwkutil.Artifact +} + +// PrepareJWKSArtifacts validates the provider's current JWKS and a replacement +// signer public key, then builds idempotent new-only and combined artifacts. +// The current artifact preserves the provider bytes exactly for recovery. +func PrepareJWKSArtifacts(currentRaw, replacementPublicPEM []byte) (PreparedJWKSArtifacts, error) { + var artifacts PreparedJWKSArtifacts + currentSet, err := jwkutil.Parse(currentRaw) + if err != nil { + return artifacts, fmt.Errorf("parse current JWKS: %w", err) + } + artifacts.Current, err = jwkutil.Inspect(currentRaw) + if err != nil { + return artifacts, fmt.Errorf("inspect current JWKS: %w", err) + } + + replacementSet, err := jwkutil.NewSigner(replacementPublicPEM) + if err != nil { + return artifacts, fmt.Errorf("build replacement JWKS: %w", err) + } + artifacts.New, err = jwkutil.Encode(replacementSet) + if err != nil { + return artifacts, fmt.Errorf("encode replacement JWKS: %w", err) + } + + combinedSet, err := jwkutil.Merge(currentSet, replacementSet) + if err != nil { + return artifacts, fmt.Errorf("merge current and replacement JWKS: %w", err) + } + artifacts.Combined, err = jwkutil.Encode(combinedSet) + if err != nil { + return artifacts, fmt.Errorf("encode combined JWKS: %w", err) + } + + return artifacts, nil +} diff --git a/pkg/cmd/provisioning/rotation/jwks_artifacts_test.go b/pkg/cmd/provisioning/rotation/jwks_artifacts_test.go new file mode 100644 index 0000000000..bc488d4f17 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/jwks_artifacts_test.go @@ -0,0 +1,134 @@ +package rotation + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "net/url" + "strings" + "testing" + + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" +) + +func TestPrepareJWKSArtifacts(t *testing.T) { + currentPEM := testPublicKeyPEM(t) + replacementPEM := testPublicKeyPEM(t) + currentSet, err := jwkutil.NewSigner(currentPEM) + if err != nil { + t.Fatalf("create current JWKS: %v", err) + } + currentEncoded, err := jwkutil.Encode(currentSet) + if err != nil { + t.Fatalf("encode current JWKS: %v", err) + } + currentRaw := append([]byte(" \n"), currentEncoded.Data...) + + artifacts, err := PrepareJWKSArtifacts(currentRaw, replacementPEM) + if err != nil { + t.Fatalf("PrepareJWKSArtifacts() returned unexpected error: %v", err) + } + if !bytes.Equal(artifacts.Current.Data, currentRaw) { + t.Fatal("current recovery artifact did not preserve provider bytes") + } + if len(artifacts.New.KeyIDs) != 1 { + t.Fatalf("new artifact key IDs = %v, want one", artifacts.New.KeyIDs) + } + if len(artifacts.Combined.KeyIDs) != 2 { + t.Fatalf("combined artifact key IDs = %v, want two", artifacts.Combined.KeyIDs) + } + if artifacts.Combined.KeyIDs[0] != currentEncoded.KeyIDs[0] || artifacts.Combined.KeyIDs[1] != artifacts.New.KeyIDs[0] { + t.Fatalf("combined key order = %v, want current then replacement", artifacts.Combined.KeyIDs) + } + + retried, err := PrepareJWKSArtifacts(artifacts.Combined.Data, replacementPEM) + if err != nil { + t.Fatalf("PrepareJWKSArtifacts(retry) returned unexpected error: %v", err) + } + if !bytes.Equal(retried.Combined.Data, artifacts.Combined.Data) { + t.Fatal("retry changed an already combined JWKS") + } +} + +func TestPrepareJWKSArtifactsRejectsInvalidInputs(t *testing.T) { + validPEM := testPublicKeyPEM(t) + validSet, err := jwkutil.NewSigner(validPEM) + if err != nil { + t.Fatalf("create valid JWKS: %v", err) + } + validArtifact, err := jwkutil.Encode(validSet) + if err != nil { + t.Fatalf("encode valid JWKS: %v", err) + } + + tests := []struct { + name string + current []byte + replacementPEM []byte + wantError string + }{ + {name: "invalid current JWKS", current: []byte(`{"keys":[]}`), replacementPEM: validPEM, wantError: "parse current JWKS"}, + {name: "invalid replacement public key", current: validArtifact.Data, replacementPEM: []byte("not PEM"), wantError: "build replacement JWKS"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := PrepareJWKSArtifacts(test.current, test.replacementPEM) + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("PrepareJWKSArtifacts() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestPrepareJWKSArtifactsPreservesSupportedCurrentMetadata(t *testing.T) { + currentSet, err := jwkutil.NewSigner(testPublicKeyPEM(t)) + if err != nil { + t.Fatalf("create current JWKS: %v", err) + } + certificateURL, err := url.Parse("https://issuer.example.test/certificate.pem") + if err != nil { + t.Fatalf("parse certificate URL: %v", err) + } + currentSet.Keys[0].CertificatesURL = certificateURL + currentSet.Keys[0].CertificateThumbprintSHA1 = bytes.Repeat([]byte{0x11}, 20) + currentSet.Keys[0].CertificateThumbprintSHA256 = bytes.Repeat([]byte{0x22}, 32) + currentArtifact, err := jwkutil.Encode(currentSet) + if err != nil { + t.Fatalf("encode current JWKS: %v", err) + } + + prepared, err := PrepareJWKSArtifacts(currentArtifact.Data, testPublicKeyPEM(t)) + if err != nil { + t.Fatalf("PrepareJWKSArtifacts() returned unexpected error: %v", err) + } + combined, err := jwkutil.Parse(prepared.Combined.Data) + if err != nil { + t.Fatalf("parse combined JWKS: %v", err) + } + current := combined.Keys[0] + if current.CertificatesURL == nil || current.CertificatesURL.String() != certificateURL.String() { + t.Fatalf("combined certificate URL = %v, want %v", current.CertificatesURL, certificateURL) + } + if !bytes.Equal(current.CertificateThumbprintSHA1, currentSet.Keys[0].CertificateThumbprintSHA1) { + t.Fatal("combined JWKS did not preserve the current SHA-1 certificate thumbprint") + } + if !bytes.Equal(current.CertificateThumbprintSHA256, currentSet.Keys[0].CertificateThumbprintSHA256) { + t.Fatal("combined JWKS did not preserve the current SHA-256 certificate thumbprint") + } +} + +func testPublicKeyPEM(t *testing.T) []byte { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + t.Fatalf("generate RSA key: %v", err) + } + publicDER, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey) + if err != nil { + t.Fatalf("marshal RSA public key: %v", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: publicDER}) +} diff --git a/pkg/cmd/provisioning/rotation/orchestrator.go b/pkg/cmd/provisioning/rotation/orchestrator.go new file mode 100644 index 0000000000..9e613c0965 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/orchestrator.go @@ -0,0 +1,938 @@ +package rotation + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "reflect" + "strings" +) + +const ( + errorCodeCancelled = "operation-cancelled" + errorCodeConflict = "state-conflict" + errorCodeExternalOutcomeUnknown = "external-outcome-unknown" + errorCodeManualInputRequired = "manual-input-required" +) + +// Orchestrator executes the shared signer-key rotation phase machine. Provider +// adapters supply target resolution and conditional publication; cluster +// actions remain behind a separately fakeable boundary. +type Orchestrator struct { + Cluster ClusterRotation + Target TargetResolver + Publisher ConditionalJWKSBackend +} + +// Run holds the local workspace lease across observation, external effects, +// public artifact persistence, and the checkpoint that follows each effect. +func (o Orchestrator) Run(ctx context.Context, options RunOptions) (RunResult, error) { + var result RunResult + if err := o.validateOptions(options); err != nil { + return result, err + } + + err := WithRotationWorkspace(options.OutputDir, func(workspace *RotationWorkspace) error { + checkpoint, existed, err := loadOrInitializeCheckpoint(workspace, options) + if err != nil { + return err + } + result.Phase = checkpoint.Phase + + if err := validateManualInput(options, checkpoint.Phase); err != nil { + return err + } + if existed { + if err := o.reconcileRecordedState(ctx, workspace, &checkpoint); err != nil { + recordCheckpointError(workspace, &checkpoint, classifyCheckpointError(err, checkpoint.Phase)) + result.Phase = checkpoint.Phase + return err + } + } + + acknowledgementConsumed := false + for checkpoint.Phase != PhaseComplete { + if err := ctx.Err(); err != nil { + recordCheckpointError(workspace, &checkpoint, errorCodeCancelled) + result.Phase = checkpoint.Phase + return err + } + if err := o.step(ctx, workspace, &checkpoint, options, &acknowledgementConsumed); err != nil { + recordCheckpointError(workspace, &checkpoint, classifyCheckpointError(err, checkpoint.Phase)) + result.Phase = checkpoint.Phase + return err + } + result.Phase = checkpoint.Phase + } + + result.Complete = true + return nil + }) + return result, err +} + +func (o Orchestrator) validateOptions(options RunOptions) error { + if o.Cluster == nil { + return fmt.Errorf("rotation cluster adapter must not be nil") + } + if o.Target == nil { + return fmt.Errorf("rotation target resolver must not be nil") + } + if !isSupportedProvider(options.Provider) { + return fmt.Errorf("unsupported rotation provider %q", options.Provider) + } + if !isSupportedPublicationMode(options.PublicationMode) { + return fmt.Errorf("unsupported rotation publication mode %q", options.PublicationMode) + } + if strings.TrimSpace(options.OutputDir) == "" { + return fmt.Errorf("rotation output directory must not be empty") + } + if options.PublicationMode == PublicationModeDirect && o.Publisher == nil { + return fmt.Errorf("direct rotation publication requires a conditional JWKS backend") + } + return nil +} + +func loadOrInitializeCheckpoint(workspace *RotationWorkspace, options RunOptions) (Checkpoint, bool, error) { + checkpoint, err := workspace.LoadCheckpoint() + switch { + case err == nil: + if !options.Resume { + return Checkpoint{}, true, fmt.Errorf("rotation checkpoint already exists in %q; use resume to continue it", options.OutputDir) + } + if checkpoint.Provider != options.Provider { + return Checkpoint{}, true, fmt.Errorf("rotation checkpoint provider %q does not match requested provider %q", checkpoint.Provider, options.Provider) + } + if checkpoint.PublicationMode != options.PublicationMode { + return Checkpoint{}, true, fmt.Errorf("rotation checkpoint publication mode %q does not match requested mode %q", checkpoint.PublicationMode, options.PublicationMode) + } + return checkpoint, true, nil + case errors.Is(err, errCheckpointNotFound): + if options.Resume { + return Checkpoint{}, false, fmt.Errorf("no rotation checkpoint exists in %q to resume", options.OutputDir) + } + outputDir, err := workspace.OutputDir() + if err != nil { + return Checkpoint{}, false, err + } + checkpoint = NewCheckpoint(options.Provider, options.PublicationMode, outputDir) + if err := workspace.SaveCheckpoint(checkpoint); err != nil { + return Checkpoint{}, false, err + } + return checkpoint, false, nil + default: + return Checkpoint{}, false, err + } +} + +func validateManualInput(options RunOptions, phase Phase) error { + hasCurrent := len(options.Manual.CurrentJWKS) != 0 + hasAcknowledgement := options.Manual.Acknowledgement != nil + if options.PublicationMode == PublicationModeDirect { + if hasCurrent || hasAcknowledgement { + return fmt.Errorf("manual rotation input cannot be used with direct publication") + } + return nil + } + if hasCurrent && hasAcknowledgement { + return fmt.Errorf("manual current JWKS input and a publication acknowledgement must be supplied in separate runs") + } + if hasCurrent && phase != PhaseInitialized && phase != PhasePreflightComplete && phase != PhaseGuardAcquired { + return fmt.Errorf("manual current JWKS input is not expected at checkpoint phase %q", phase) + } + if !hasAcknowledgement { + return nil + } + + expectedPhase := Phase("") + switch phase { + case PhaseCombinedJWKSBuilt: + expectedPhase = PhaseCombinedJWKSPublished + case PhasePostRebootStable: + expectedPhase = PhaseNewOnlyJWKSPublished + default: + return fmt.Errorf("manual publication acknowledgement is not expected at checkpoint phase %q", phase) + } + if options.Manual.Acknowledgement.Phase != expectedPhase { + return fmt.Errorf("manual publication acknowledgement for phase %q cannot be used at checkpoint phase %q", options.Manual.Acknowledgement.Phase, phase) + } + return nil +} + +func (o Orchestrator) step(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint, options RunOptions, acknowledgementConsumed *bool) error { + if phaseAtLeast(checkpoint.Phase, PhaseGuardAcquired) && !phaseAtLeast(checkpoint.Phase, PhaseGuardReleaseRecorded) { + if err := o.requireRotationGuardHeld(ctx, *checkpoint); err != nil { + return err + } + } + switch checkpoint.Phase { + case PhaseInitialized: + return o.completePreflight(ctx, workspace, checkpoint, options.PublicationMode) + case PhasePreflightComplete: + return o.acquireRotationGuard(ctx, workspace, checkpoint) + case PhaseGuardAcquired: + return o.captureCurrentJWKS(ctx, workspace, checkpoint, options) + case PhaseCurrentJWKSRead: + return o.requestReplacement(ctx, workspace, checkpoint) + case PhaseNextKeyRequested: + return o.captureReplacementSigner(ctx, workspace, checkpoint) + case PhaseNextPublicKeyRead: + return buildNewJWKS(workspace, checkpoint) + case PhaseNewJWKSBuilt: + return buildCombinedJWKS(workspace, checkpoint) + case PhaseCombinedJWKSBuilt: + return o.publishArtifact(ctx, workspace, checkpoint, options, acknowledgementConsumed, PhaseCombinedJWKSPublished, ArtifactCombinedJWKS, ArtifactCurrentJWKS) + case PhaseCombinedJWKSPublished: + return o.waitForSignerRollout(ctx, workspace, checkpoint) + case PhaseSignerRolloutStable: + return o.recordRebootIntent(ctx, workspace, checkpoint) + case PhaseRebootIntentRecorded: + return o.completeReboot(ctx, workspace, checkpoint) + case PhaseNodesRebooted: + return o.waitForPostRebootStability(ctx, workspace, checkpoint) + case PhasePostRebootStable: + return o.publishArtifact(ctx, workspace, checkpoint, options, acknowledgementConsumed, PhaseNewOnlyJWKSPublished, ArtifactNewJWKS, ArtifactCombinedJWKS) + case PhaseNewOnlyJWKSPublished: + if err := o.reconcileRecordedState(ctx, workspace, checkpoint); err != nil { + return err + } + return advanceCheckpoint(workspace, checkpoint, PhaseGuardReleaseRecorded, nil) + case PhaseGuardReleaseRecorded: + return o.releaseRotationGuard(ctx, workspace, checkpoint) + default: + return fmt.Errorf("cannot execute unsupported rotation phase %q", checkpoint.Phase) + } +} + +func (o Orchestrator) completePreflight(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint, mode PublicationMode) error { + preflight, err := o.Cluster.Preflight(ctx) + if err != nil { + return fmt.Errorf("rotation cluster preflight failed: %w", err) + } + if err := validateOpaqueCheckpointValue("cluster identity", preflight.ClusterIdentity); err != nil { + return err + } + signerState, err := o.observeStableSignerState(ctx, checkpoint.Phase, nil) + if err != nil { + return fmt.Errorf("capture stable pre-rotation public signer baseline: %w", err) + } + if signerState.Reference == nil { + return &ConflictError{Phase: checkpoint.Phase, Reason: "the pre-rotation signer Secret is absent"} + } + + targetIdentity, err := o.Target.ResolveTarget(ctx) + if err != nil { + return fmt.Errorf("resolve rotation publication target: %w", err) + } + if err := validateOpaqueCheckpointValue("target identity", targetIdentity); err != nil { + return err + } + if mode == PublicationModeDirect { + if err := o.Publisher.CheckAccess(ctx, targetIdentity); err != nil { + return fmt.Errorf("validate direct publication access: %w", err) + } + } + + baseline := clonePublicSignerBaseline(signerState.Baseline) + reference := *signerState.Reference + guard, err := deriveRotationGuardReference(preflight.ClusterIdentity, checkpoint.Provider, targetIdentity, baseline, reference) + if err != nil { + return fmt.Errorf("derive signer-rotation guard reference: %w", err) + } + return advanceCheckpoint(workspace, checkpoint, PhasePreflightComplete, func(next *Checkpoint) { + next.ClusterIdentity = preflight.ClusterIdentity + next.TargetIdentity = targetIdentity + next.PreRotationSignerBaseline = &baseline + next.PreRotationSignerRef = &reference + next.RotationGuard = &guard + }) +} + +func (o Orchestrator) captureCurrentJWKS(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint, options RunOptions) error { + var current []byte + if options.PublicationMode == PublicationModeManual { + if len(options.Manual.CurrentJWKS) == 0 { + return &PauseError{Phase: checkpoint.Phase, Reason: PauseForCurrentJWKS, Artifact: ArtifactCurrentJWKS} + } + current = append([]byte(nil), options.Manual.CurrentJWKS...) + } else { + observed, err := o.Publisher.ReadJWKS(ctx, checkpoint.TargetIdentity) + if err != nil { + return fmt.Errorf("read current provider JWKS: %w", err) + } + current = append([]byte(nil), observed.Data...) + } + + metadata, err := workspace.WriteArtifact(ArtifactCurrentJWKS, current) + if err != nil { + return err + } + return advanceCheckpoint(workspace, checkpoint, PhaseCurrentJWKSRead, func(next *Checkpoint) { + next.Artifacts = append(next.Artifacts, metadata) + }) +} + +func (o Orchestrator) requestReplacement(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + guard := *checkpoint.RotationGuard + observed, err := o.observeStableSignerState(ctx, checkpoint.Phase, checkpoint.RotationGuard) + if err != nil { + return fmt.Errorf("observe signer state before replacement request: %w", err) + } + progress, _, err := classifySignerState(checkpoint.Phase, observed, *checkpoint) + if err != nil { + return err + } + if progress != signerStateOriginal { + return advanceCheckpoint(workspace, checkpoint, PhaseNextKeyRequested, nil) + } + if checkpoint.PublicationMode == PublicationModeDirect { + current, err := workspace.ReadArtifact(ArtifactCurrentJWKS) + if err != nil { + return err + } + providerState, err := o.Publisher.ReadJWKS(ctx, checkpoint.TargetIdentity) + if err != nil { + return fmt.Errorf("read provider JWKS before signer replacement: %w", err) + } + if !bytes.Equal(providerState.Data, current.Data) { + return &ConflictError{ + Phase: checkpoint.Phase, + Reason: fmt.Sprintf("provider JWKS digest %s does not match the recorded current artifact before signer replacement", publicDigest(providerState.Data)), + } + } + } + + outcome, requestErr := o.Cluster.RequestReplacement(ctx, guard, *checkpoint.PreRotationSignerRef) + if err := validateEffectOutcome(outcome); err != nil { + return err + } + reconciled, observeErr := o.observeStableSignerState(ctx, checkpoint.Phase, checkpoint.RotationGuard) + if observeErr == nil { + progress, _, err = classifySignerState(checkpoint.Phase, reconciled, *checkpoint) + if err != nil { + return err + } + if progress != signerStateOriginal { + return advanceCheckpoint(workspace, checkpoint, PhaseNextKeyRequested, nil) + } + } + + switch outcome { + case EffectUnknown, EffectSubmitted: + cause := requestErr + if cause == nil { + cause = observeErr + } + return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "request signer replacement", Cause: cause} + case EffectNotApplied: + if requestErr != nil { + return fmt.Errorf("signer replacement request was not applied: %w", requestErr) + } + if observeErr != nil { + return fmt.Errorf("reconcile unapplied signer replacement request: %w", observeErr) + } + return &ConflictError{Phase: checkpoint.Phase, Reason: "the preconditioned signer deletion was not applied and the original signer is still present"} + default: + return fmt.Errorf("unsupported signer replacement outcome %q", outcome) + } +} + +func (o Orchestrator) captureReplacementSigner(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + guard := *checkpoint.RotationGuard + if err := o.Cluster.WaitForReplacement(ctx, guard, *checkpoint.PreRotationSignerRef); err != nil { + return fmt.Errorf("wait for replacement signer public key: %w", err) + } + observed, err := o.observeStableSignerState(ctx, checkpoint.Phase, checkpoint.RotationGuard) + if err != nil { + return fmt.Errorf("observe stable replacement signer state: %w", err) + } + progress, replacement, err := classifySignerState(checkpoint.Phase, observed, *checkpoint) + if err != nil { + return err + } + if progress != signerStateReplacementReady || replacement == nil { + return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "wait for replacement signer", Cause: fmt.Errorf("replacement Secret and exactly one appended public signer are not both observable")} + } + + metadata, err := workspace.WriteArtifact(ArtifactReplacementPublicKey, replacement.PublicPEM) + if err != nil { + return err + } + evidence := cloneReplacementSignerEvidence(replacement.Evidence) + return advanceCheckpoint(workspace, checkpoint, PhaseNextPublicKeyRead, func(next *Checkpoint) { + next.Artifacts = append(next.Artifacts, metadata) + next.ReplacementSigner = &evidence + }) +} + +func buildNewJWKS(workspace *RotationWorkspace, checkpoint *Checkpoint) error { + prepared, err := prepareFromRecordedArtifacts(workspace) + if err != nil { + return err + } + metadata, err := workspace.WriteArtifact(ArtifactNewJWKS, prepared.New.Data) + if err != nil { + return err + } + return advanceCheckpoint(workspace, checkpoint, PhaseNewJWKSBuilt, func(next *Checkpoint) { + next.Artifacts = append(next.Artifacts, metadata) + }) +} + +func buildCombinedJWKS(workspace *RotationWorkspace, checkpoint *Checkpoint) error { + prepared, err := prepareFromRecordedArtifacts(workspace) + if err != nil { + return err + } + metadata, err := workspace.WriteArtifact(ArtifactCombinedJWKS, prepared.Combined.Data) + if err != nil { + return err + } + return advanceCheckpoint(workspace, checkpoint, PhaseCombinedJWKSBuilt, func(next *Checkpoint) { + next.Artifacts = append(next.Artifacts, metadata) + }) +} + +func prepareFromRecordedArtifacts(workspace *RotationWorkspace) (PreparedJWKSArtifacts, error) { + current, err := workspace.ReadArtifact(ArtifactCurrentJWKS) + if err != nil { + return PreparedJWKSArtifacts{}, err + } + replacement, err := workspace.ReadArtifact(ArtifactReplacementPublicKey) + if err != nil { + return PreparedJWKSArtifacts{}, err + } + return PrepareJWKSArtifacts(current.Data, replacement.Data) +} + +func (o Orchestrator) publishArtifact(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint, options RunOptions, acknowledgementConsumed *bool, publishedPhase Phase, artifactName, predecessorName string) error { + if err := o.revalidateRecordedSignerEvidence(ctx, *checkpoint); err != nil { + return fmt.Errorf("revalidate replacement signer before publication: %w", err) + } + desired, err := workspace.ReadArtifact(artifactName) + if err != nil { + return err + } + if options.PublicationMode == PublicationModeManual { + if err := validateAndConsumeAcknowledgement(options.Manual.Acknowledgement, acknowledgementConsumed, publishedPhase, desired); err != nil { + return err + } + } else { + predecessor, err := workspace.ReadArtifact(predecessorName) + if err != nil { + return err + } + if err := o.publishDirect(ctx, *checkpoint, desired, predecessor); err != nil { + return err + } + } + + confirmation := PublicationConfirmation{Phase: publishedPhase, Artifact: artifactName, SHA256: desired.Metadata.SHA256} + return advanceCheckpoint(workspace, checkpoint, publishedPhase, func(next *Checkpoint) { + next.Publications = append(next.Publications, confirmation) + }) +} + +func validateAndConsumeAcknowledgement(acknowledgement *ManualAcknowledgement, consumed *bool, expectedPhase Phase, desired StoredArtifact) error { + if acknowledgement == nil || *consumed { + return &PauseError{ + Phase: phaseBefore(expectedPhase), + Reason: PauseForPublication, + Artifact: desired.Metadata.Name, + Path: desired.Path, + SHA256: desired.Metadata.SHA256, + } + } + if acknowledgement.Phase != expectedPhase || acknowledgement.Artifact != desired.Metadata.Name || acknowledgement.SHA256 != desired.Metadata.SHA256 { + return &ConflictError{Phase: phaseBefore(expectedPhase), Reason: "the manual acknowledgement does not match the required phase, artifact, and SHA-256 digest"} + } + *consumed = true + return nil +} + +func (o Orchestrator) publishDirect(ctx context.Context, checkpoint Checkpoint, desired, predecessor StoredArtifact) error { + observed, err := o.Publisher.ReadJWKS(ctx, checkpoint.TargetIdentity) + if err != nil { + return fmt.Errorf("read provider JWKS before conditional publication: %w", err) + } + if bytes.Equal(observed.Data, desired.Data) { + return nil + } + if !bytes.Equal(observed.Data, predecessor.Data) { + return &ConflictError{Phase: checkpoint.Phase, Reason: fmt.Sprintf("provider JWKS digest %s is neither the recorded predecessor nor the desired artifact", publicDigest(observed.Data))} + } + if strings.TrimSpace(observed.Revision) == "" { + return fmt.Errorf("provider JWKS read did not return the revision required for conditional publication") + } + + outcome, publishErr := o.Publisher.PublishIfVersion(ctx, checkpoint.TargetIdentity, observed.Revision, append([]byte(nil), desired.Data...)) + if err := validateEffectOutcome(outcome); err != nil { + return err + } + readback, readbackErr := o.Publisher.ReadJWKS(ctx, checkpoint.TargetIdentity) + if readbackErr == nil && bytes.Equal(readback.Data, desired.Data) { + return nil + } + + switch outcome { + case EffectUnknown, EffectSubmitted: + cause := publishErr + if cause == nil { + cause = readbackErr + } + return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "conditionally publish provider JWKS", Cause: cause} + case EffectNotApplied: + if publishErr != nil { + return fmt.Errorf("conditional provider JWKS publication was not applied: %w", publishErr) + } + if readbackErr != nil { + return fmt.Errorf("read provider JWKS after rejected conditional publication: %w", readbackErr) + } + return &ConflictError{Phase: checkpoint.Phase, Reason: fmt.Sprintf("conditional provider write was rejected and the target now has digest %s", publicDigest(readback.Data))} + default: + return fmt.Errorf("unsupported provider publication outcome %q", outcome) + } +} + +func (o Orchestrator) waitForSignerRollout(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + replacementKeyID, err := recordedReplacementKeyID(workspace) + if err != nil { + return err + } + guard := *checkpoint.RotationGuard + if err := o.Cluster.WaitForSignerRollout(ctx, guard, replacementKeyID); err != nil { + return fmt.Errorf("wait for replacement signer rollout: %w", err) + } + if err := o.revalidateRecordedSignerEvidence(ctx, *checkpoint); err != nil { + return fmt.Errorf("revalidate replacement signer after rollout: %w", err) + } + if checkpoint.PublicationMode == PublicationModeDirect { + combined, err := workspace.ReadArtifact(ArtifactCombinedJWKS) + if err != nil { + return err + } + observed, err := o.Publisher.ReadJWKS(ctx, checkpoint.TargetIdentity) + if err != nil { + return fmt.Errorf("read provider JWKS before node reboot: %w", err) + } + if !bytes.Equal(observed.Data, combined.Data) { + return &ConflictError{Phase: checkpoint.Phase, Reason: fmt.Sprintf("provider JWKS digest %s does not match the recorded combined artifact before node reboot", publicDigest(observed.Data))} + } + } + return advanceCheckpoint(workspace, checkpoint, PhaseSignerRolloutStable, nil) +} + +func (o Orchestrator) recordRebootIntent(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + replacementKeyID, err := recordedReplacementKeyID(workspace) + if err != nil { + return err + } + operationID, err := rebootIntentID(checkpoint.ClusterIdentity, replacementKeyID) + if err != nil { + return fmt.Errorf("derive node reboot operation ID: %w", err) + } + observation, err := o.observeCanonicalReboot(ctx, *checkpoint.RotationGuard, operationID) + if err != nil { + return fmt.Errorf("observe existing node reboot record: %w", err) + } + + var intent RebootIntent + if observation.CanonicalIntent != nil { + intent = cloneRebootIntent(*observation.CanonicalIntent) + } else { + plan, err := o.Cluster.PrepareReboot(ctx, *checkpoint.RotationGuard, replacementKeyID) + if err != nil { + return fmt.Errorf("prepare node reboot intent: %w", err) + } + intent, err = buildRebootIntent(checkpoint.ClusterIdentity, replacementKeyID, plan) + if err != nil { + return fmt.Errorf("validate node reboot intent: %w", err) + } + } + return advanceCheckpoint(workspace, checkpoint, PhaseRebootIntentRecorded, func(next *Checkpoint) { + next.RebootIntent = &intent + }) +} + +func (o Orchestrator) completeReboot(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + status, intent, err := o.observeAndAdoptCanonicalReboot(ctx, workspace, checkpoint) + if err != nil { + return fmt.Errorf("observe persisted node reboot intent: %w", err) + } + if status == RebootComplete { + return advanceCheckpoint(workspace, checkpoint, PhaseNodesRebooted, nil) + } + + if status == RebootNotStarted { + outcome, requestErr := o.Cluster.RequestReboot(ctx, *checkpoint.RotationGuard, cloneRebootIntent(intent)) + if err := validateEffectOutcome(outcome); err != nil { + return err + } + status, intent, err = o.observeAndAdoptCanonicalReboot(ctx, workspace, checkpoint) + if err != nil { + if outcome == EffectUnknown || outcome == EffectSubmitted { + return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "request persisted node reboot", Cause: errors.Join(requestErr, err)} + } + return fmt.Errorf("reconcile node reboot request: %w", err) + } + if status == RebootComplete { + return advanceCheckpoint(workspace, checkpoint, PhaseNodesRebooted, nil) + } + if status == RebootNotStarted { + switch outcome { + case EffectUnknown, EffectSubmitted: + return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "request persisted node reboot", Cause: requestErr} + case EffectNotApplied: + if requestErr != nil { + return fmt.Errorf("node reboot request was not applied: %w", requestErr) + } + return &ConflictError{Phase: checkpoint.Phase, Reason: "the persisted reboot request was not applied"} + } + } + } + + if err := o.Cluster.WaitForReboot(ctx, *checkpoint.RotationGuard, cloneRebootIntent(intent)); err != nil { + return fmt.Errorf("wait for persisted node reboot: %w", err) + } + status, intent, err = o.observeAndAdoptCanonicalReboot(ctx, workspace, checkpoint) + if err != nil { + return fmt.Errorf("confirm persisted node reboot completion: %w", err) + } + if status != RebootComplete { + return &ConflictError{Phase: checkpoint.Phase, Reason: fmt.Sprintf("reboot wait returned before intent %q was complete", intent.ID)} + } + return advanceCheckpoint(workspace, checkpoint, PhaseNodesRebooted, nil) +} + +func (o Orchestrator) observeCanonicalReboot(ctx context.Context, guard RotationGuardReference, operationID string) (RebootObservation, error) { + observation, err := o.Cluster.ObserveReboot(ctx, guard, operationID) + if err != nil { + return RebootObservation{}, err + } + if err := validateRebootObservation(operationID, observation); err != nil { + return RebootObservation{}, err + } + if observation.CanonicalIntent != nil { + intent := cloneRebootIntent(*observation.CanonicalIntent) + observation.CanonicalIntent = &intent + } + return observation, nil +} + +func (o Orchestrator) observeAndAdoptCanonicalReboot(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) (RebootStatus, RebootIntent, error) { + localIntent := cloneRebootIntent(*checkpoint.RebootIntent) + observation, err := o.observeCanonicalReboot(ctx, *checkpoint.RotationGuard, localIntent.ID) + if err != nil { + return "", RebootIntent{}, err + } + if observation.CanonicalIntent == nil || reflect.DeepEqual(localIntent, *observation.CanonicalIntent) { + return observation.Status, localIntent, nil + } + if checkpoint.Phase != PhaseRebootIntentRecorded { + return "", RebootIntent{}, &ConflictError{ + Phase: checkpoint.Phase, + Reason: fmt.Sprintf("cluster reboot record %q differs from the checkpoint after reboot completion was recorded", localIntent.ID), + } + } + + canonicalIntent := cloneRebootIntent(*observation.CanonicalIntent) + next := cloneCheckpoint(*checkpoint) + next.RebootIntent = &canonicalIntent + next.LastErrorCode = "" + if err := workspace.saveCheckpointAdoptingCanonicalRebootIntent(next); err != nil { + return "", RebootIntent{}, fmt.Errorf("persist cluster-canonical reboot intent: %w", err) + } + *checkpoint = next + return observation.Status, cloneRebootIntent(canonicalIntent), nil +} + +func (o Orchestrator) waitForPostRebootStability(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + intent := cloneRebootIntent(*checkpoint.RebootIntent) + if err := o.Cluster.WaitForPostRebootStable(ctx, *checkpoint.RotationGuard, intent); err != nil { + return fmt.Errorf("wait for post-reboot cluster stability: %w", err) + } + return advanceCheckpoint(workspace, checkpoint, PhasePostRebootStable, nil) +} + +func (o Orchestrator) reconcileRecordedState(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + if checkpoint.Phase == PhaseInitialized { + return nil + } + if checkpoint.Phase == PhasePreflightComplete { + // The exact reference is durable but the external guard may not have + // been acquired yet. The next phase reconciles that mutation before + // any guarded signer observation. + return nil + } + if checkpoint.Phase == PhaseGuardReleaseRecorded || checkpoint.Phase == PhaseComplete { + // Once exact terminal completion is observable, signer and provider + // state may legitimately belong to a later rotation. + return o.reconcileReleasedRotationGuard(ctx, *checkpoint) + } + if err := o.requireRotationGuardHeld(ctx, *checkpoint); err != nil { + return err + } + guard := *checkpoint.RotationGuard + + targetIdentity, err := o.Target.ResolveTarget(ctx) + if err != nil { + return fmt.Errorf("resolve rotation publication target during reconciliation: %w", err) + } + if targetIdentity != checkpoint.TargetIdentity { + return &ConflictError{Phase: checkpoint.Phase, Reason: fmt.Sprintf("resolved target identity %q does not match checkpoint target %q", targetIdentity, checkpoint.TargetIdentity)} + } + if checkpoint.PublicationMode == PublicationModeDirect && checkpoint.Phase != PhaseComplete { + if err := o.Publisher.CheckAccess(ctx, targetIdentity); err != nil { + return fmt.Errorf("validate direct publication access during reconciliation: %w", err) + } + } + if checkpoint.RebootIntent != nil { + if _, _, err := o.observeAndAdoptCanonicalReboot(ctx, workspace, checkpoint); err != nil { + return fmt.Errorf("reconcile cluster-canonical reboot intent: %w", err) + } + } + signerState, err := o.observeStableSignerState(ctx, checkpoint.Phase, checkpoint.RotationGuard) + if err != nil { + return fmt.Errorf("reconcile signer state: %w", err) + } + if err := validateRecordedSignerState(checkpoint.Phase, signerState, *checkpoint); err != nil { + return err + } + + expectation, err := clusterExpectation(workspace, *checkpoint) + if err != nil { + return err + } + if err := o.Cluster.Reconcile(ctx, guard, expectation); err != nil { + return fmt.Errorf("reconcile cluster state at phase %q: %w", checkpoint.Phase, err) + } + + if checkpoint.PublicationMode != PublicationModeDirect { + return nil + } + artifactName := "" + switch { + case phaseAtLeast(checkpoint.Phase, PhaseNewOnlyJWKSPublished): + artifactName = ArtifactNewJWKS + case checkpoint.Phase == PhasePostRebootStable: + combined, err := workspace.ReadArtifact(ArtifactCombinedJWKS) + if err != nil { + return err + } + newOnly, err := workspace.ReadArtifact(ArtifactNewJWKS) + if err != nil { + return err + } + observed, err := o.Publisher.ReadJWKS(ctx, checkpoint.TargetIdentity) + if err != nil { + return fmt.Errorf("read provider JWKS during final-publication reconciliation: %w", err) + } + if !bytes.Equal(observed.Data, combined.Data) && !bytes.Equal(observed.Data, newOnly.Data) { + return &ConflictError{Phase: checkpoint.Phase, Reason: fmt.Sprintf("provider JWKS digest %s is neither the recorded combined predecessor nor the new-only artifact", publicDigest(observed.Data))} + } + return nil + case phaseAtLeast(checkpoint.Phase, PhaseCombinedJWKSPublished): + artifactName = ArtifactCombinedJWKS + default: + return nil + } + expected, err := workspace.ReadArtifact(artifactName) + if err != nil { + return err + } + observed, err := o.Publisher.ReadJWKS(ctx, checkpoint.TargetIdentity) + if err != nil { + return fmt.Errorf("read provider JWKS during reconciliation: %w", err) + } + if !bytes.Equal(observed.Data, expected.Data) { + return &ConflictError{Phase: checkpoint.Phase, Reason: fmt.Sprintf("provider JWKS digest %s does not match recorded artifact %q", publicDigest(observed.Data), artifactName)} + } + return nil +} + +func clusterExpectation(workspace *RotationWorkspace, checkpoint Checkpoint) (ClusterExpectation, error) { + expectation := ClusterExpectation{ + Phase: checkpoint.Phase, + ClusterIdentity: checkpoint.ClusterIdentity, + RotationGuard: *checkpoint.RotationGuard, + PreRotationSignerBaseline: clonePublicSignerBaseline(*checkpoint.PreRotationSignerBaseline), + PreRotationSignerRef: *checkpoint.PreRotationSignerRef, + } + if phaseAtLeast(checkpoint.Phase, PhaseNextPublicKeyRead) { + replacementKeyID, err := recordedReplacementKeyID(workspace) + if err != nil { + return ClusterExpectation{}, err + } + if replacementKeyID != checkpoint.ReplacementSigner.Entry.KeyID { + return ClusterExpectation{}, fmt.Errorf("recorded replacement public key does not match checkpoint replacement evidence") + } + evidence := cloneReplacementSignerEvidence(*checkpoint.ReplacementSigner) + expectation.ReplacementSigner = &evidence + } + if checkpoint.RebootIntent != nil { + intent := cloneRebootIntent(*checkpoint.RebootIntent) + expectation.RebootIntent = &intent + } + return expectation, nil +} +func recordedReplacementKeyID(workspace *RotationWorkspace) (string, error) { + replacement, err := workspace.ReadArtifact(ArtifactReplacementPublicKey) + if err != nil { + return "", err + } + if len(replacement.Metadata.KeyIDs) != 1 { + return "", fmt.Errorf("replacement signer artifact must record exactly one key ID") + } + return replacement.Metadata.KeyIDs[0], nil +} + +func validateEffectOutcome(outcome EffectOutcome) error { + switch outcome { + case EffectSubmitted, EffectNotApplied, EffectUnknown: + return nil + default: + return fmt.Errorf("adapter returned unsupported external effect outcome %q", outcome) + } +} + +func validateRebootStatus(status RebootStatus) error { + switch status { + case RebootNotStarted, RebootInProgress, RebootComplete: + return nil + default: + return fmt.Errorf("adapter returned unsupported reboot status %q", status) + } +} + +func validateRebootObservation(operationID string, observation RebootObservation) error { + if err := validateOpaqueCheckpointValue("reboot operation ID", operationID); err != nil { + return err + } + if err := validateRebootStatus(observation.Status); err != nil { + return err + } + if observation.Status == RebootNotStarted { + if observation.CanonicalIntent != nil { + return fmt.Errorf("adapter returned a canonical reboot intent for status %q", RebootNotStarted) + } + return nil + } + if observation.CanonicalIntent == nil { + return fmt.Errorf("adapter returned reboot status %q without the canonical reboot intent", observation.Status) + } + if err := validateRebootIntent(*observation.CanonicalIntent); err != nil { + return fmt.Errorf("adapter returned an invalid canonical reboot intent: %w", err) + } + if observation.CanonicalIntent.ID != operationID { + return fmt.Errorf("adapter returned canonical reboot intent %q for operation %q", observation.CanonicalIntent.ID, operationID) + } + return nil +} + +func phaseBefore(phase Phase) Phase { + position := phasePosition(phase) + if position <= 0 { + return "" + } + return orderedPhases[position-1] +} + +func advanceCheckpoint(workspace *RotationWorkspace, checkpoint *Checkpoint, nextPhase Phase, mutate func(*Checkpoint)) error { + next := cloneCheckpoint(*checkpoint) + next.Phase = nextPhase + next.LastErrorCode = "" + if mutate != nil { + mutate(&next) + } + if err := workspace.SaveCheckpoint(next); err != nil { + return err + } + *checkpoint = next + return nil +} + +func recordCheckpointError(workspace *RotationWorkspace, checkpoint *Checkpoint, code string) { + if code == "" || checkpoint.Phase == "" { + return + } + next := cloneCheckpoint(*checkpoint) + next.LastErrorCode = code + if err := workspace.SaveCheckpoint(next); err == nil { + *checkpoint = next + } +} + +func classifyCheckpointError(err error, phase Phase) string { + var pause *PauseError + if errors.As(err, &pause) { + return errorCodeManualInputRequired + } + var unknown *OutcomeUnknownError + if errors.As(err, &unknown) { + return errorCodeExternalOutcomeUnknown + } + var conflict *ConflictError + if errors.As(err, &conflict) { + return errorCodeConflict + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return errorCodeCancelled + } + if phase == "" { + return "operation-failed" + } + return string(phase) + "-failed" +} + +func cloneCheckpoint(checkpoint Checkpoint) Checkpoint { + clone := checkpoint + clone.Artifacts = make([]ArtifactMetadata, len(checkpoint.Artifacts)) + for index, artifact := range checkpoint.Artifacts { + clone.Artifacts[index] = artifact + clone.Artifacts[index].KeyIDs = append([]string(nil), artifact.KeyIDs...) + } + clone.Publications = append([]PublicationConfirmation(nil), checkpoint.Publications...) + if checkpoint.PreRotationSignerBaseline != nil { + baseline := clonePublicSignerBaseline(*checkpoint.PreRotationSignerBaseline) + clone.PreRotationSignerBaseline = &baseline + } + if checkpoint.PreRotationSignerRef != nil { + reference := *checkpoint.PreRotationSignerRef + clone.PreRotationSignerRef = &reference + } + if checkpoint.RotationGuard != nil { + reference := *checkpoint.RotationGuard + clone.RotationGuard = &reference + } + if checkpoint.ReplacementSigner != nil { + evidence := cloneReplacementSignerEvidence(*checkpoint.ReplacementSigner) + clone.ReplacementSigner = &evidence + } + if checkpoint.RebootIntent != nil { + intent := cloneRebootIntent(*checkpoint.RebootIntent) + clone.RebootIntent = &intent + } + return clone +} + +func cloneRebootIntent(intent RebootIntent) RebootIntent { + intent.Targets = append([]string(nil), intent.Targets...) + intent.Baselines = append([]NodeRebootBaseline(nil), intent.Baselines...) + return intent +} + +func clonePublicSignerBaseline(baseline PublicSignerBaseline) PublicSignerBaseline { + baseline.Entries = append([]PublicSignerBaselineEntry(nil), baseline.Entries...) + return baseline +} + +func cloneReplacementSignerEvidence(evidence ReplacementSignerEvidence) ReplacementSignerEvidence { + return evidence +} + +func publicDigest(data []byte) string { + digest := sha256.Sum256(data) + return fmt.Sprintf("%x", digest) +} diff --git a/pkg/cmd/provisioning/rotation/orchestrator_safety_test.go b/pkg/cmd/provisioning/rotation/orchestrator_safety_test.go new file mode 100644 index 0000000000..c54e664ffa --- /dev/null +++ b/pkg/cmd/provisioning/rotation/orchestrator_safety_test.go @@ -0,0 +1,882 @@ +package rotation + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" +) + +func TestOrchestratorDoesNotAdvanceUntilSubmittedReplacementIsObserved(t *testing.T) { + oldPublic := testPublicKeyPEM(t) + newPublic := testPublicKeyPEM(t) + oldSigner := safetySigner(oldPublic, "old-uid", "10") + newSigner := safetySigner(newPublic, "new-uid", "11") + currentJWKS := encodedJWKSForTest(t, oldPublic) + + cluster := newSafetyCluster(oldSigner, newSigner) + cluster.replacementOutcome = EffectSubmitted + cluster.applyReplacement = false + publisher := newSafetyPublisher(currentJWKS) + engine := Orchestrator{ + Cluster: cluster, + Target: safetyTarget("aws://issuer/keys.json"), + Publisher: publisher, + } + outputDir := filepath.Join(t.TempDir(), "rotation") + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := engine.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhaseCurrentJWKSRead || cluster.replacementRequests != 1 { + t.Fatalf("first Run() result = %#v, replacement requests = %d", result, cluster.replacementRequests) + } + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load checkpoint after unknown replacement request: %v", err) + } + if checkpoint.LastErrorCode != errorCodeExternalOutcomeUnknown { + t.Fatalf("last error code = %q, want %q", checkpoint.LastErrorCode, errorCodeExternalOutcomeUnknown) + } + if cluster.replacementReference != *checkpoint.PreRotationSignerRef { + t.Fatalf("replacement request reference = %#v, want immutable checkpoint reference %#v", cluster.replacementReference, *checkpoint.PreRotationSignerRef) + } + + cluster.nextReference = nil + options.Resume = true + result, err = engine.Run(context.Background(), options) + if !errors.As(err, &unknown) { + t.Fatalf("resume during Secret recreation error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhaseNextKeyRequested || cluster.replacementRequests != 1 { + t.Fatalf("resume during Secret recreation result = %#v, replacement requests = %d", result, cluster.replacementRequests) + } + + cluster.applySafetyReplacement() + result, err = engine.Run(context.Background(), options) + if err != nil { + t.Fatalf("second resume Run() returned unexpected error: %v", err) + } + if !result.Complete || result.Phase != PhaseComplete { + t.Fatalf("resume Run() result = %#v, want complete", result) + } + if cluster.replacementRequests != 1 { + t.Fatalf("replacement request count = %d, want exactly one", cluster.replacementRequests) + } +} + +func TestOrchestratorResumeDoesNotRepeatUnknownRebootRequest(t *testing.T) { + oldPublic := testPublicKeyPEM(t) + newPublic := testPublicKeyPEM(t) + oldSigner := safetySigner(oldPublic, "old-uid", "10") + newSigner := safetySigner(newPublic, "new-uid", "11") + cluster := newSafetyCluster(oldSigner, newSigner) + cluster.replacementOutcome = EffectSubmitted + cluster.rebootOutcome = EffectUnknown + cluster.rebootStatuses = []RebootStatus{RebootNotStarted, RebootNotStarted, RebootNotStarted} + publisher := newSafetyPublisher(encodedJWKSForTest(t, oldPublic)) + engine := Orchestrator{ + Cluster: cluster, + Target: safetyTarget("aws://issuer/keys.json"), + Publisher: publisher, + } + outputDir := filepath.Join(t.TempDir(), "rotation") + cluster.beforeRebootRequest = func(intent RebootIntent) error { + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + return err + } + if checkpoint.Phase != PhaseRebootIntentRecorded || checkpoint.RebootIntent == nil || checkpoint.RebootIntent.ID != intent.ID { + return fmt.Errorf("reboot request ran before matching intent was durable: %#v", checkpoint) + } + return nil + } + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := engine.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhaseRebootIntentRecorded || cluster.rebootRequests != 1 { + t.Fatalf("first Run() result = %#v, reboot requests = %d", result, cluster.rebootRequests) + } + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load checkpoint after unknown reboot request: %v", err) + } + if checkpoint.RebootIntent == nil || checkpoint.RebootIntent.ID == "" { + t.Fatalf("persisted reboot intent = %#v, want a deterministic ID", checkpoint.RebootIntent) + } + + cluster.rebootStatuses = []RebootStatus{RebootComplete} + options.Resume = true + result, err = engine.Run(context.Background(), options) + if err != nil { + t.Fatalf("resume Run() returned unexpected error: %v", err) + } + if !result.Complete || cluster.rebootRequests != 1 { + t.Fatalf("resume result = %#v, reboot request count = %d", result, cluster.rebootRequests) + } +} + +func TestOrchestratorDoesNotExposeMutableRebootIntentAliases(t *testing.T) { + oldPublic := testPublicKeyPEM(t) + newPublic := testPublicKeyPEM(t) + base := newSafetyCluster( + safetySigner(oldPublic, "old-uid", "10"), + safetySigner(newPublic, "new-uid", "11"), + ) + base.rebootStatuses = []RebootStatus{RebootNotStarted, RebootNotStarted, RebootInProgress, RebootComplete} + cluster := &mutatingRebootCluster{t: t, safetyCluster: base} + publisher := newSafetyPublisher(encodedJWKSForTest(t, oldPublic)) + engine := Orchestrator{ + Cluster: cluster, + Target: safetyTarget("aws://issuer/keys.json"), + Publisher: publisher, + } + + result, err := engine.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: filepath.Join(t.TempDir(), "rotation"), + }) + if err != nil { + t.Fatalf("Run() returned unexpected error: %v", err) + } + if !result.Complete { + t.Fatalf("Run() result = %#v, want complete", result) + } +} + +func TestBuildRebootIntentIsDeterministicAcrossWorkspaces(t *testing.T) { + replacementKeyID, err := safetySignerKeyID(safetySigner(testPublicKeyPEM(t), "uid", "1")) + if err != nil { + t.Fatalf("derive replacement key ID: %v", err) + } + first, err := buildRebootIntent("cluster-123", replacementKeyID, RebootPlan{ + Targets: []string{"worker", "master"}, + Baselines: []NodeRebootBaseline{ + {Target: "worker", Node: "worker-0", BootID: "old-worker-boot"}, + {Target: "master", Node: "master-0", BootID: "old-master-boot"}, + }, + }) + if err != nil { + t.Fatalf("build first reboot intent: %v", err) + } + second, err := buildRebootIntent("cluster-123", replacementKeyID, RebootPlan{ + Targets: []string{"master", "worker"}, + Baselines: []NodeRebootBaseline{ + {Target: "master", Node: "master-0", BootID: "newer-master-baseline"}, + {Target: "worker", Node: "worker-0", BootID: "newer-worker-baseline"}, + }, + }) + if err != nil { + t.Fatalf("build second reboot intent: %v", err) + } + if first.ID != second.ID { + t.Fatalf("reboot intent IDs differ across plan order/baseline time: %q and %q", first.ID, second.ID) + } + if !reflect.DeepEqual(first.Targets, []string{"master", "worker"}) { + t.Fatalf("canonical targets = %v", first.Targets) + } + + differentSnapshot, err := buildRebootIntent("cluster-123", replacementKeyID, RebootPlan{ + Targets: []string{"worker"}, + Baselines: []NodeRebootBaseline{{Target: "worker", Node: "worker-0", BootID: "old-worker-boot"}}, + }) + if err != nil { + t.Fatalf("build different-snapshot reboot intent: %v", err) + } + if first.ID != differentSnapshot.ID { + t.Fatalf("one signer rotation produced different reboot IDs across target snapshots: %q and %q", first.ID, differentSnapshot.ID) + } +} + +func TestValidateRebootObservationRequiresCanonicalRecord(t *testing.T) { + canonical := testNodeRebootIntent() + invalidCanonical := cloneRebootIntent(*canonical) + invalidCanonical.Baselines = nil + tests := []struct { + name string + operation string + observed RebootObservation + wantError string + }{ + {name: "not started without record", operation: canonical.ID, observed: RebootObservation{Status: RebootNotStarted}}, + {name: "in progress with canonical record", operation: canonical.ID, observed: RebootObservation{Status: RebootInProgress, CanonicalIntent: canonical}}, + {name: "not started with record", operation: canonical.ID, observed: RebootObservation{Status: RebootNotStarted, CanonicalIntent: canonical}, wantError: "canonical reboot intent"}, + {name: "in progress without record", operation: canonical.ID, observed: RebootObservation{Status: RebootInProgress}, wantError: "without the canonical"}, + {name: "mismatched operation", operation: "other-operation", observed: RebootObservation{Status: RebootComplete, CanonicalIntent: canonical}, wantError: "for operation"}, + {name: "invalid canonical record", operation: canonical.ID, observed: RebootObservation{Status: RebootComplete, CanonicalIntent: &invalidCanonical}, wantError: "invalid canonical"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := validateRebootObservation(test.operation, test.observed) + if test.wantError == "" { + if err != nil { + t.Fatalf("validateRebootObservation() returned unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("validateRebootObservation() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestOrchestratorTwoWorkspacesAdoptOneClusterCanonicalReboot(t *testing.T) { + oldPublic := testPublicKeyPEM(t) + newPublic := testPublicKeyPEM(t) + oldSigner := safetySigner(oldPublic, "old-uid", "10") + newSigner := safetySigner(newPublic, "new-uid", "11") + currentJWKS := encodedJWKSForTest(t, oldPublic) + registry := &sharedRebootRegistry{} + stopBeforeRebootIntent := errors.New("stop before recording the reboot intent") + + firstCluster := &sharedRegistryCluster{ + safetyCluster: newSafetyCluster(oldSigner, newSigner), + registry: registry, + prepareErr: stopBeforeRebootIntent, + plan: RebootPlan{ + Targets: []string{"worker", "master"}, + Baselines: []NodeRebootBaseline{ + {Target: "worker", Node: "worker-0", BootID: "worker-before-first-request"}, + {Target: "master", Node: "master-0", BootID: "master-before-first-request"}, + }, + }, + } + secondCluster := &sharedRegistryCluster{ + safetyCluster: newSafetyCluster(oldSigner, newSigner), + registry: registry, + prepareErr: stopBeforeRebootIntent, + plan: RebootPlan{ + Targets: []string{"worker"}, + Baselines: []NodeRebootBaseline{ + {Target: "worker", Node: "worker-0", BootID: "worker-after-snapshot-drift"}, + }, + }, + } + firstPublisher := newSafetyPublisher(currentJWKS) + secondPublisher := newSafetyPublisher(currentJWKS) + firstEngine := Orchestrator{Cluster: firstCluster, Target: safetyTarget("aws://issuer/keys.json"), Publisher: firstPublisher} + secondEngine := Orchestrator{Cluster: secondCluster, Target: safetyTarget("aws://issuer/keys.json"), Publisher: secondPublisher} + firstOptions := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: filepath.Join(t.TempDir(), "first")} + secondOptions := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: filepath.Join(t.TempDir(), "second")} + + for _, run := range []struct { + name string + engine Orchestrator + options RunOptions + }{ + {name: "first workspace", engine: firstEngine, options: firstOptions}, + {name: "second workspace", engine: secondEngine, options: secondOptions}, + } { + result, err := run.engine.Run(context.Background(), run.options) + if !errors.Is(err, stopBeforeRebootIntent) { + t.Fatalf("%s initial Run() error = %v, want %v", run.name, err, stopBeforeRebootIntent) + } + if result.Phase != PhaseSignerRolloutStable { + t.Fatalf("%s initial Run() phase = %q, want %q", run.name, result.Phase, PhaseSignerRolloutStable) + } + } + + firstCluster.prepareErr = nil + secondCluster.prepareErr = nil + firstProposal := recordRebootProposalForTest(t, firstEngine, firstOptions.OutputDir) + secondProposal := recordRebootProposalForTest(t, secondEngine, secondOptions.OutputDir) + if firstProposal.RebootIntent.ID != secondProposal.RebootIntent.ID { + t.Fatalf("workspace operation IDs differ: %q and %q", firstProposal.RebootIntent.ID, secondProposal.RebootIntent.ID) + } + if reflect.DeepEqual(firstProposal.RebootIntent, secondProposal.RebootIntent) { + t.Fatal("workspace reboot proposals unexpectedly match; test requires snapshot drift") + } + + firstOptions.Resume = true + firstResult, err := firstEngine.Run(context.Background(), firstOptions) + if err != nil || !firstResult.Complete { + t.Fatalf("first workspace resume result = %#v, error = %v", firstResult, err) + } + secondOptions.Resume = true + secondResult, err := secondEngine.Run(context.Background(), secondOptions) + if err != nil || !secondResult.Complete { + t.Fatalf("second workspace resume result = %#v, error = %v", secondResult, err) + } + + canonical, requestCalls, requestMutations := registry.snapshot() + if requestCalls != 1 || requestMutations != 1 { + t.Fatalf("cluster reboot request calls = %d, mutations = %d; want exactly one of each", requestCalls, requestMutations) + } + if canonical == nil || !reflect.DeepEqual(*canonical, *firstProposal.RebootIntent) { + t.Fatalf("cluster-canonical reboot intent = %#v, want first proposal %#v", canonical, firstProposal.RebootIntent) + } + for _, outputDir := range []string{firstOptions.OutputDir, secondOptions.OutputDir} { + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("load completed checkpoint %q: %v", outputDir, err) + } + if checkpoint.Phase != PhaseComplete || !reflect.DeepEqual(checkpoint.RebootIntent, canonical) { + t.Fatalf("completed checkpoint %q has phase %q and reboot intent %#v, want canonical %#v", outputDir, checkpoint.Phase, checkpoint.RebootIntent, canonical) + } + } + if secondCluster.canonicalReconciliations == 0 { + t.Fatal("second workspace did not reconcile the adopted canonical reboot intent") + } +} + +func recordRebootProposalForTest(t *testing.T, engine Orchestrator, outputDir string) Checkpoint { + t.Helper() + var checkpoint Checkpoint + err := WithRotationWorkspace(outputDir, func(workspace *RotationWorkspace) error { + var err error + checkpoint, err = workspace.LoadCheckpoint() + if err != nil { + return err + } + return engine.recordRebootIntent(context.Background(), workspace, &checkpoint) + }) + if err != nil { + t.Fatalf("record reboot proposal in %q: %v", outputDir, err) + } + if checkpoint.Phase != PhaseRebootIntentRecorded || checkpoint.RebootIntent == nil { + t.Fatalf("reboot proposal checkpoint = %#v", checkpoint) + } + return checkpoint +} + +func TestOrchestratorResumesFinalPublicationAfterAppliedWriteAndFailedReadback(t *testing.T) { + oldPublic := testPublicKeyPEM(t) + newPublic := testPublicKeyPEM(t) + oldSigner := safetySigner(oldPublic, "old-uid", "10") + newSigner := safetySigner(newPublic, "new-uid", "11") + cluster := newSafetyCluster(oldSigner, newSigner) + publisher := newSafetyPublisher(encodedJWKSForTest(t, oldPublic)) + // Reads 1-4 capture and recheck the current state and confirm combined + // publication, and read 5 rechecks overlap before reboot. Read 6 is the + // final predecessor read; read 7 fails after the final write. + publisher.failReadNumber = 7 + engine := Orchestrator{ + Cluster: cluster, + Target: safetyTarget("aws://issuer/keys.json"), + Publisher: publisher, + } + outputDir := filepath.Join(t.TempDir(), "rotation") + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := engine.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhasePostRebootStable || publisher.publishCalls != 2 { + t.Fatalf("first Run() result = %#v, publication calls = %d", result, publisher.publishCalls) + } + + options.Resume = true + result, err = engine.Run(context.Background(), options) + if err != nil { + t.Fatalf("resume Run() returned unexpected error: %v", err) + } + if !result.Complete || publisher.publishCalls != 2 { + t.Fatalf("resume result = %#v, publication calls = %d; final write was repeated", result, publisher.publishCalls) + } +} + +func TestPublishDirectRequiresExpectedPredecessorAndExactReadback(t *testing.T) { + current := StoredArtifact{Data: []byte("current")} + combined := StoredArtifact{Data: []byte("combined")} + newOnly := StoredArtifact{Data: []byte("new-only")} + checkpoint := Checkpoint{Phase: PhaseCombinedJWKSBuilt, TargetIdentity: "target"} + + t.Run("unrelated provider state is never overwritten", func(t *testing.T) { + publisher := &safetyPublisher{state: VersionedJWKS{Data: []byte("unrelated"), Revision: "7"}, publishOutcome: EffectSubmitted, applyPublication: true} + engine := Orchestrator{Publisher: publisher} + err := engine.publishDirect(context.Background(), checkpoint, combined, current) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("publishDirect() error = %v, want ConflictError", err) + } + if publisher.publishCalls != 0 { + t.Fatalf("conditional publication calls = %d, want zero", publisher.publishCalls) + } + }) + + t.Run("new-only state cannot satisfy combined publication", func(t *testing.T) { + publisher := &safetyPublisher{state: VersionedJWKS{Data: newOnly.Data, Revision: "7"}, publishOutcome: EffectSubmitted, applyPublication: true} + engine := Orchestrator{Publisher: publisher} + err := engine.publishDirect(context.Background(), checkpoint, combined, current) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("publishDirect() error = %v, want ConflictError", err) + } + if publisher.publishCalls != 0 { + t.Fatalf("conditional publication calls = %d, want zero", publisher.publishCalls) + } + }) + + t.Run("unknown write is accepted only after exact readback", func(t *testing.T) { + publisher := &safetyPublisher{ + reads: []VersionedJWKS{ + {Data: current.Data, Revision: "7"}, + {Data: combined.Data, Revision: "8"}, + }, + publishOutcome: EffectUnknown, + publishErr: errors.New("transport ended before response"), + } + engine := Orchestrator{Publisher: publisher} + if err := engine.publishDirect(context.Background(), checkpoint, combined, current); err != nil { + t.Fatalf("publishDirect() returned unexpected error after exact readback: %v", err) + } + }) + + t.Run("unknown write with unchanged readback remains resumable", func(t *testing.T) { + publisher := &safetyPublisher{ + reads: []VersionedJWKS{ + {Data: current.Data, Revision: "7"}, + {Data: current.Data, Revision: "7"}, + }, + publishOutcome: EffectUnknown, + publishErr: errors.New("transport ended before response"), + } + engine := Orchestrator{Publisher: publisher} + err := engine.publishDirect(context.Background(), checkpoint, combined, current) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("publishDirect() error = %v, want OutcomeUnknownError", err) + } + }) +} + +type safetyTarget string + +func (target safetyTarget) ResolveTarget(context.Context) (string, error) { + return string(target), nil +} + +type sharedRebootRegistry struct { + mu sync.Mutex + canonical *RebootIntent + requestCalls int + requestMutations int +} + +func (registry *sharedRebootRegistry) observe(operationID string) (RebootObservation, error) { + registry.mu.Lock() + defer registry.mu.Unlock() + if registry.canonical == nil { + return RebootObservation{Status: RebootNotStarted}, nil + } + if registry.canonical.ID != operationID { + return RebootObservation{}, fmt.Errorf("canonical operation ID %q does not match requested ID %q", registry.canonical.ID, operationID) + } + canonical := cloneRebootIntent(*registry.canonical) + return RebootObservation{Status: RebootComplete, CanonicalIntent: &canonical}, nil +} + +func (registry *sharedRebootRegistry) request(intent RebootIntent) (EffectOutcome, error) { + registry.mu.Lock() + defer registry.mu.Unlock() + registry.requestCalls++ + if registry.canonical == nil { + canonical := cloneRebootIntent(intent) + registry.canonical = &canonical + registry.requestMutations++ + return EffectSubmitted, nil + } + if registry.canonical.ID != intent.ID { + return EffectNotApplied, fmt.Errorf("canonical operation ID %q does not match requested ID %q", registry.canonical.ID, intent.ID) + } + return EffectSubmitted, nil +} + +func (registry *sharedRebootRegistry) snapshot() (*RebootIntent, int, int) { + registry.mu.Lock() + defer registry.mu.Unlock() + if registry.canonical == nil { + return nil, registry.requestCalls, registry.requestMutations + } + canonical := cloneRebootIntent(*registry.canonical) + return &canonical, registry.requestCalls, registry.requestMutations +} + +type sharedRegistryCluster struct { + *safetyCluster + registry *sharedRebootRegistry + plan RebootPlan + prepareErr error + canonicalReconciliations int +} + +func (cluster *sharedRegistryCluster) Reconcile(ctx context.Context, guard RotationGuardReference, expectation ClusterExpectation) error { + canonical, _, _ := cluster.registry.snapshot() + if canonical != nil && expectation.RebootIntent != nil { + if !reflect.DeepEqual(*expectation.RebootIntent, *canonical) { + return fmt.Errorf("generic reconciliation received stale reboot intent %#v, want canonical %#v", expectation.RebootIntent, canonical) + } + cluster.canonicalReconciliations++ + } + return cluster.safetyCluster.Reconcile(ctx, guard, expectation) +} + +func (cluster *sharedRegistryCluster) PrepareReboot(context.Context, RotationGuardReference, string) (RebootPlan, error) { + if cluster.prepareErr != nil { + return RebootPlan{}, cluster.prepareErr + } + return RebootPlan{ + Targets: append([]string(nil), cluster.plan.Targets...), + Baselines: append([]NodeRebootBaseline(nil), cluster.plan.Baselines...), + }, nil +} + +func (cluster *sharedRegistryCluster) ObserveReboot(_ context.Context, _ RotationGuardReference, operationID string) (RebootObservation, error) { + return cluster.registry.observe(operationID) +} + +func (cluster *sharedRegistryCluster) RequestReboot(_ context.Context, _ RotationGuardReference, intent RebootIntent) (EffectOutcome, error) { + return cluster.registry.request(intent) +} + +func (cluster *sharedRegistryCluster) WaitForReboot(context.Context, RotationGuardReference, RebootIntent) error { + return nil +} + +func (cluster *sharedRegistryCluster) WaitForPostRebootStable(context.Context, RotationGuardReference, RebootIntent) error { + return nil +} + +type safetyCluster struct { + preflight ClusterPreflight + nextReference *SignerObjectReference + nextBundle PublicSignerBundleObservation + replacement safetySignerFixture + + replacementOutcome EffectOutcome + replacementErr error + replacementRequests int + replacementReference SignerObjectReference + applyReplacement bool + + rebootIntent RebootIntent + canonicalReboot *RebootIntent + rebootStatuses []RebootStatus + rebootOutcome EffectOutcome + rebootErr error + rebootRequests int + beforeRebootRequest func(RebootIntent) error + + reconcileCalls int + + guardReference *RotationGuardReference + completedGuards map[string]struct{} + guardAcquireCalls int + guardAcquireMutations int + guardReleaseCalls int + guardReleaseMutations int +} + +func newSafetyCluster(oldSigner, replacement safetySignerFixture) *safetyCluster { + return &safetyCluster{ + preflight: ClusterPreflight{ClusterIdentity: "cluster-123"}, + nextReference: cloneSignerReferenceForTest(&oldSigner.Reference), + nextBundle: PublicSignerBundleObservation{ + ConfigMapUID: "safety-signer-configmap-uid", + ConfigMapResourceVersion: "1000", + Signers: []PublicSignerObservation{clonePublicSignerObservationForTest(oldSigner.Signer)}, + }, + replacement: replacement, + replacementOutcome: EffectSubmitted, + applyReplacement: true, + rebootIntent: RebootIntent{ + ID: "rotation-cluster-123-new-key", + Targets: []string{"worker", "master"}, + Baselines: []NodeRebootBaseline{ + {Target: "worker", Node: "worker-0", BootID: "worker-boot-old"}, + {Target: "master", Node: "master-0", BootID: "master-boot-old"}, + }, + }, + rebootStatuses: []RebootStatus{RebootComplete}, + rebootOutcome: EffectSubmitted, + } +} + +func (cluster *safetyCluster) Preflight(context.Context) (ClusterPreflight, error) { + return ClusterPreflight{ + ClusterIdentity: cluster.preflight.ClusterIdentity, + }, nil +} + +func (cluster *safetyCluster) ObserveRotationGuard(_ context.Context, reference RotationGuardReference) (RotationGuardObservation, error) { + if _, completed := cluster.completedGuards[reference.OperationID]; completed { + return RotationGuardObservation{Status: RotationGuardCompleted, OperationID: reference.OperationID}, nil + } + if cluster.guardReference == nil { + return RotationGuardObservation{Status: RotationGuardNotFound}, nil + } + if *cluster.guardReference == reference { + return RotationGuardObservation{Status: RotationGuardHeld, OperationID: reference.OperationID}, nil + } + return RotationGuardObservation{Status: RotationGuardOwnedByOther, OperationID: cluster.guardReference.OperationID}, nil +} + +func (cluster *safetyCluster) AcquireRotationGuard(_ context.Context, reference RotationGuardReference) (EffectOutcome, error) { + cluster.guardAcquireCalls++ + if cluster.guardReference == nil { + stored := reference + cluster.guardReference = &stored + cluster.guardAcquireMutations++ + return EffectSubmitted, nil + } + if *cluster.guardReference == reference { + return EffectSubmitted, nil + } + return EffectNotApplied, nil +} + +func (cluster *safetyCluster) ReleaseRotationGuard(_ context.Context, reference RotationGuardReference) (EffectOutcome, error) { + cluster.guardReleaseCalls++ + if _, completed := cluster.completedGuards[reference.OperationID]; completed { + return EffectSubmitted, nil + } + if cluster.guardReference == nil || *cluster.guardReference != reference { + return EffectNotApplied, nil + } + cluster.guardReference = nil + if cluster.completedGuards == nil { + cluster.completedGuards = make(map[string]struct{}) + } + cluster.completedGuards[reference.OperationID] = struct{}{} + cluster.guardReleaseMutations++ + return EffectSubmitted, nil +} + +func (cluster *safetyCluster) Reconcile(context.Context, RotationGuardReference, ClusterExpectation) error { + cluster.reconcileCalls++ + return nil +} + +func (cluster *safetyCluster) ObserveSignerReference(context.Context, *RotationGuardReference) (*SignerObjectReference, error) { + return cloneSignerReferenceForTest(cluster.nextReference), nil +} + +func (cluster *safetyCluster) ObservePublicSignerBundle(context.Context, *RotationGuardReference) (PublicSignerBundleObservation, error) { + return clonePublicSignerBundleForTest(cluster.nextBundle), nil +} + +func (cluster *safetyCluster) RequestReplacement(_ context.Context, _ RotationGuardReference, reference SignerObjectReference) (EffectOutcome, error) { + cluster.replacementRequests++ + cluster.replacementReference = reference + if cluster.replacementOutcome == EffectSubmitted && cluster.replacementErr == nil && cluster.applyReplacement { + cluster.applySafetyReplacement() + } + return cluster.replacementOutcome, cluster.replacementErr +} + +func (cluster *safetyCluster) WaitForReplacement(context.Context, RotationGuardReference, SignerObjectReference) error { + return nil +} + +func (cluster *safetyCluster) applySafetyReplacement() { + cluster.nextReference = cloneSignerReferenceForTest(&cluster.replacement.Reference) + cluster.nextBundle.ConfigMapResourceVersion = "1001" + for _, signer := range cluster.nextBundle.Signers { + if signer.Name == cluster.replacement.Signer.Name { + return + } + } + cluster.nextBundle.Signers = append(cluster.nextBundle.Signers, clonePublicSignerObservationForTest(cluster.replacement.Signer)) +} + +func (cluster *safetyCluster) WaitForSignerRollout(context.Context, RotationGuardReference, string) error { + return nil +} + +func (cluster *safetyCluster) PrepareReboot(context.Context, RotationGuardReference, string) (RebootPlan, error) { + intent := cloneRebootIntent(cluster.rebootIntent) + return RebootPlan{Targets: intent.Targets, Baselines: intent.Baselines}, nil +} + +func (cluster *safetyCluster) ObserveReboot(_ context.Context, _ RotationGuardReference, operationID string) (RebootObservation, error) { + status := RebootComplete + if len(cluster.rebootStatuses) == 0 { + status = RebootComplete + } else { + status = cluster.rebootStatuses[0] + cluster.rebootStatuses = cluster.rebootStatuses[1:] + } + if status == RebootNotStarted { + return RebootObservation{Status: status}, nil + } + intent := cloneRebootIntent(cluster.rebootIntent) + if cluster.canonicalReboot != nil { + intent = cloneRebootIntent(*cluster.canonicalReboot) + } else { + intent.ID = operationID + } + return RebootObservation{Status: status, CanonicalIntent: &intent}, nil +} + +func (cluster *safetyCluster) RequestReboot(_ context.Context, _ RotationGuardReference, intent RebootIntent) (EffectOutcome, error) { + cluster.rebootRequests++ + if cluster.beforeRebootRequest != nil { + if err := cluster.beforeRebootRequest(intent); err != nil { + return EffectNotApplied, err + } + } + if cluster.rebootOutcome == EffectSubmitted && cluster.rebootErr == nil && cluster.canonicalReboot == nil { + canonical := cloneRebootIntent(intent) + cluster.canonicalReboot = &canonical + } + return cluster.rebootOutcome, cluster.rebootErr +} + +func (cluster *safetyCluster) WaitForReboot(context.Context, RotationGuardReference, RebootIntent) error { + return nil +} + +func (cluster *safetyCluster) WaitForPostRebootStable(context.Context, RotationGuardReference, RebootIntent) error { + return nil +} + +type mutatingRebootCluster struct { + *safetyCluster + t *testing.T + want *RebootIntent +} + +func (cluster *mutatingRebootCluster) ObserveReboot(ctx context.Context, guard RotationGuardReference, operationID string) (RebootObservation, error) { + return cluster.safetyCluster.ObserveReboot(ctx, guard, operationID) +} + +func (cluster *mutatingRebootCluster) RequestReboot(ctx context.Context, guard RotationGuardReference, intent RebootIntent) (EffectOutcome, error) { + original := cloneRebootIntent(intent) + cluster.assertAndMutateIntent(intent) + return cluster.safetyCluster.RequestReboot(ctx, guard, original) +} + +func (cluster *mutatingRebootCluster) WaitForReboot(ctx context.Context, guard RotationGuardReference, intent RebootIntent) error { + cluster.assertAndMutateIntent(intent) + return cluster.safetyCluster.WaitForReboot(ctx, guard, intent) +} + +func (cluster *mutatingRebootCluster) WaitForPostRebootStable(ctx context.Context, guard RotationGuardReference, intent RebootIntent) error { + cluster.assertAndMutateIntent(intent) + return cluster.safetyCluster.WaitForPostRebootStable(ctx, guard, intent) +} + +func (cluster *mutatingRebootCluster) assertAndMutateIntent(intent RebootIntent) { + cluster.t.Helper() + if cluster.want == nil { + want := cloneRebootIntent(intent) + cluster.want = &want + } else if !reflect.DeepEqual(intent, *cluster.want) { + cluster.t.Errorf("adapter received mutated reboot intent = %#v, want %#v", intent, *cluster.want) + } + if len(intent.Targets) != 0 { + intent.Targets[0] = "tampered-target" + } + if len(intent.Baselines) != 0 { + intent.Baselines[0].BootID = "tampered-boot-id" + } +} + +type safetySignerFixture struct { + Signer PublicSignerObservation + Reference SignerObjectReference +} + +func safetySigner(publicKey []byte, uid, resourceVersion string) safetySignerFixture { + name := "service-account-001.pub" + if uid != "old-uid" { + name = "service-account-002.pub" + } + return safetySignerFixture{ + Signer: PublicSignerObservation{ + Name: name, + PublicKeyPEM: append([]byte(nil), publicKey...), + }, + Reference: SignerObjectReference{UID: uid, ResourceVersion: resourceVersion}, + } +} + +func safetySignerKeyID(signer safetySignerFixture) (string, error) { + state, err := normalizePublicSignerBundle(PublicSignerBundleObservation{ + ConfigMapUID: "test-configmap-uid", + ConfigMapResourceVersion: "1", + Signers: []PublicSignerObservation{signer.Signer}, + }) + if err != nil { + return "", err + } + return state.Baseline.Entries[0].KeyID, nil +} + +type safetyPublisher struct { + state VersionedJWKS + reads []VersionedJWKS + readCalls int + failReadNumber int + + publishOutcome EffectOutcome + publishErr error + applyPublication bool + publishCalls int +} + +func newSafetyPublisher(current []byte) *safetyPublisher { + return &safetyPublisher{ + state: VersionedJWKS{Data: append([]byte(nil), current...), Revision: "1"}, + publishOutcome: EffectSubmitted, + applyPublication: true, + } +} + +func (publisher *safetyPublisher) CheckAccess(context.Context, string) error { + return nil +} + +func (publisher *safetyPublisher) ReadJWKS(context.Context, string) (VersionedJWKS, error) { + publisher.readCalls++ + if publisher.failReadNumber == publisher.readCalls { + return VersionedJWKS{}, errors.New("provider readback unavailable") + } + if len(publisher.reads) != 0 { + read := publisher.reads[0] + publisher.reads = publisher.reads[1:] + return VersionedJWKS{Data: append([]byte(nil), read.Data...), Revision: read.Revision}, nil + } + return VersionedJWKS{Data: append([]byte(nil), publisher.state.Data...), Revision: publisher.state.Revision}, nil +} + +func (publisher *safetyPublisher) PublishIfVersion(_ context.Context, _ string, revision string, data []byte) (EffectOutcome, error) { + publisher.publishCalls++ + if publisher.applyPublication { + if revision != publisher.state.Revision { + return EffectNotApplied, fmt.Errorf("revision changed") + } + publisher.state.Data = append([]byte(nil), data...) + publisher.state.Revision = fmt.Sprintf("%d", publisher.publishCalls+1) + } + return publisher.publishOutcome, publisher.publishErr +} + +func TestPauseErrorDoesNotExposeJWKSContents(t *testing.T) { + err := (&PauseError{ + Phase: PhaseCombinedJWKSBuilt, + Reason: PauseForPublication, + Artifact: ArtifactCombinedJWKS, + Path: "/tmp/public/jwks.combined.json", + SHA256: testDigest, + }).Error() + if strings.Contains(err, `"keys"`) || strings.Contains(err, "/tmp/public") { + t.Fatalf("PauseError exposed artifact contents or local path: %q", err) + } +} diff --git a/pkg/cmd/provisioning/rotation/orchestrator_test.go b/pkg/cmd/provisioning/rotation/orchestrator_test.go new file mode 100644 index 0000000000..2732a2b4b6 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/orchestrator_test.go @@ -0,0 +1,813 @@ +package rotation + +import ( + "bytes" + "context" + "errors" + "fmt" + "reflect" + "testing" +) + +func TestOrchestratorDirectHappyPathOrdersExternalEffects(t *testing.T) { + t.Parallel() + + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + outputDir := t.TempDir() + + result, err := harness.orchestrator.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: outputDir, + }) + if err != nil { + t.Fatalf("Run() returned unexpected error: %v", err) + } + if result.Phase != PhaseComplete || !result.Complete { + t.Fatalf("Run() result = %+v, want complete phase", result) + } + + wantEvents := []string{ + "cluster.preflight", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "target.resolve", + "publisher.check-access", + "cluster.observe-rotation-guard", + "cluster.acquire-rotation-guard", + "cluster.observe-rotation-guard", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "cluster.observe-rotation-guard", + "publisher.read:current", + "cluster.observe-rotation-guard", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "publisher.read:current", + "cluster.request-replacement", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "cluster.observe-rotation-guard", + "cluster.wait-for-replacement", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "cluster.observe-rotation-guard", + "cluster.observe-rotation-guard", + "cluster.observe-rotation-guard", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "publisher.read:current", + "publisher.publish:combined", + "publisher.read:combined", + "cluster.observe-rotation-guard", + "cluster.wait-for-signer-rollout", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "publisher.read:combined", + "cluster.observe-rotation-guard", + "cluster.observe-reboot:not-started", + "cluster.prepare-reboot", + "cluster.observe-rotation-guard", + "cluster.observe-reboot:not-started", + "cluster.request-reboot", + "cluster.observe-reboot:in-progress", + "cluster.wait-for-reboot", + "cluster.observe-reboot:complete", + "cluster.observe-rotation-guard", + "cluster.wait-for-post-reboot-stable", + "cluster.observe-rotation-guard", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "publisher.read:combined", + "publisher.publish:new-only", + "publisher.read:new-only", + "cluster.observe-rotation-guard", + "cluster.observe-rotation-guard", + "target.resolve", + "publisher.check-access", + "cluster.observe-reboot:complete", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "cluster.reconcile:new-only-jwks-published", + "publisher.read:new-only", + "cluster.observe-rotation-guard", + "cluster.release-rotation-guard", + "cluster.observe-rotation-guard", + } + if !reflect.DeepEqual(harness.events, wantEvents) { + t.Fatalf("external effect order =\n%q\nwant\n%q", harness.events, wantEvents) + } + + if harness.cluster.replacementRequests != 1 { + t.Fatalf("replacement requests = %d, want 1", harness.cluster.replacementRequests) + } + if harness.cluster.rebootRequests != 1 { + t.Fatalf("reboot requests = %d, want 1", harness.cluster.rebootRequests) + } + if !reflect.DeepEqual(harness.publisher.publications, []string{"combined", "new-only"}) { + t.Fatalf("provider publications = %v, want combined then new-only", harness.publisher.publications) + } + if harness.cluster.requestedReference != harness.initialReference { + t.Fatalf("replacement precondition = %+v, want %+v", harness.cluster.requestedReference, harness.initialReference) + } + if harness.cluster.waitedReference != harness.initialReference { + t.Fatalf("replacement wait reference = %+v, want %+v", harness.cluster.waitedReference, harness.initialReference) + } + if harness.cluster.rolloutKeyID != harness.replacementKeyID { + t.Fatalf("rollout key ID = %q, want %q", harness.cluster.rolloutKeyID, harness.replacementKeyID) + } + if harness.cluster.rebootKeyID != harness.replacementKeyID { + t.Fatalf("reboot key ID = %q, want %q", harness.cluster.rebootKeyID, harness.replacementKeyID) + } + + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("LoadCheckpoint() returned unexpected error: %v", err) + } + wantArtifactNames := []string{ + ArtifactCurrentJWKS, + ArtifactReplacementPublicKey, + ArtifactNewJWKS, + ArtifactCombinedJWKS, + } + gotArtifactNames := make([]string, 0, len(checkpoint.Artifacts)) + for _, artifact := range checkpoint.Artifacts { + gotArtifactNames = append(gotArtifactNames, artifact.Name) + } + if !reflect.DeepEqual(gotArtifactNames, wantArtifactNames) { + t.Fatalf("checkpoint artifact order = %v, want %v", gotArtifactNames, wantArtifactNames) + } + wantPublications := []PublicationConfirmation{ + {Phase: PhaseCombinedJWKSPublished, Artifact: ArtifactCombinedJWKS, SHA256: harness.combinedSHA256}, + {Phase: PhaseNewOnlyJWKSPublished, Artifact: ArtifactNewJWKS, SHA256: harness.newOnlySHA256}, + } + if !reflect.DeepEqual(checkpoint.Publications, wantPublications) { + t.Fatalf("checkpoint publications = %+v, want %+v", checkpoint.Publications, wantPublications) + } +} + +func TestOrchestratorManualPublicationRequiresDigestBoundAcknowledgementsAcrossResumes(t *testing.T) { + t.Parallel() + + harness := newOrchestratorTestHarness(t, PublicationModeManual) + outputDir := t.TempDir() + baseOptions := RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeManual, + OutputDir: outputDir, + } + + result, err := harness.orchestrator.Run(context.Background(), baseOptions) + currentPause := requireOrchestratorPause(t, err, PhaseGuardAcquired, PauseForCurrentJWKS, ArtifactCurrentJWKS) + if result.Phase != PhaseGuardAcquired || result.Complete { + t.Fatalf("initial Run() result = %+v, want paused after guard acquisition", result) + } + if currentPause.SHA256 != "" { + t.Fatalf("current-JWKS pause digest = %q, want empty digest", currentPause.SHA256) + } + + withCurrent := baseOptions + withCurrent.Resume = true + withCurrent.Manual.CurrentJWKS = append([]byte(nil), harness.currentJWKS...) + result, err = harness.orchestrator.Run(context.Background(), withCurrent) + combinedPause := requireOrchestratorPause(t, err, PhaseCombinedJWKSBuilt, PauseForPublication, ArtifactCombinedJWKS) + if result.Phase != PhaseCombinedJWKSBuilt || result.Complete { + t.Fatalf("Run() with current JWKS result = %+v, want combined-JWKS pause", result) + } + if combinedPause.SHA256 != harness.combinedSHA256 { + t.Fatalf("combined-JWKS pause digest = %q, want %q", combinedPause.SHA256, harness.combinedSHA256) + } + + wrongAcknowledgement := baseOptions + wrongAcknowledgement.Resume = true + wrongAcknowledgement.Manual.Acknowledgement = &ManualAcknowledgement{ + Phase: PhaseCombinedJWKSPublished, + Artifact: combinedPause.Artifact, + SHA256: combinedPause.SHA256 + "-different", + } + result, err = harness.orchestrator.Run(context.Background(), wrongAcknowledgement) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() with wrong digest error = %v, want ConflictError", err) + } + if result.Phase != PhaseCombinedJWKSBuilt { + t.Fatalf("Run() with wrong digest phase = %q, want %q", result.Phase, PhaseCombinedJWKSBuilt) + } + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("LoadCheckpoint() after rejected acknowledgement returned error: %v", loadErr) + } + if checkpoint.Phase != PhaseCombinedJWKSBuilt || len(checkpoint.Publications) != 0 { + t.Fatalf("checkpoint advanced after rejected acknowledgement: %+v", checkpoint) + } + + acknowledgeCombined := baseOptions + acknowledgeCombined.Resume = true + acknowledgeCombined.Manual.Acknowledgement = &ManualAcknowledgement{ + Phase: PhaseCombinedJWKSPublished, + Artifact: combinedPause.Artifact, + SHA256: combinedPause.SHA256, + } + result, err = harness.orchestrator.Run(context.Background(), acknowledgeCombined) + newOnlyPause := requireOrchestratorPause(t, err, PhasePostRebootStable, PauseForPublication, ArtifactNewJWKS) + if result.Phase != PhasePostRebootStable || result.Complete { + t.Fatalf("Run() after combined acknowledgement result = %+v, want new-only pause", result) + } + if newOnlyPause.SHA256 != harness.newOnlySHA256 { + t.Fatalf("new-only pause digest = %q, want %q", newOnlyPause.SHA256, harness.newOnlySHA256) + } + + acknowledgeNewOnly := baseOptions + acknowledgeNewOnly.Resume = true + acknowledgeNewOnly.Manual.Acknowledgement = &ManualAcknowledgement{ + Phase: PhaseNewOnlyJWKSPublished, + Artifact: newOnlyPause.Artifact, + SHA256: newOnlyPause.SHA256, + } + result, err = harness.orchestrator.Run(context.Background(), acknowledgeNewOnly) + if err != nil { + t.Fatalf("Run() after new-only acknowledgement returned unexpected error: %v", err) + } + if result.Phase != PhaseComplete || !result.Complete { + t.Fatalf("final Run() result = %+v, want complete phase", result) + } + + wantReconciledPhases := []Phase{ + PhaseGuardAcquired, + PhaseCombinedJWKSBuilt, + PhaseCombinedJWKSBuilt, + PhasePostRebootStable, + PhaseNewOnlyJWKSPublished, + } + if !reflect.DeepEqual(harness.cluster.reconciledPhases(), wantReconciledPhases) { + t.Fatalf("reconciled phases = %v, want %v", harness.cluster.reconciledPhases(), wantReconciledPhases) + } + checkpoint, err = LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("LoadCheckpoint() returned unexpected error: %v", err) + } + if checkpoint.LastErrorCode != "" { + t.Fatalf("completed checkpoint last error code = %q, want empty", checkpoint.LastErrorCode) + } + wantPublications := []PublicationConfirmation{ + {Phase: PhaseCombinedJWKSPublished, Artifact: ArtifactCombinedJWKS, SHA256: harness.combinedSHA256}, + {Phase: PhaseNewOnlyJWKSPublished, Artifact: ArtifactNewJWKS, SHA256: harness.newOnlySHA256}, + } + if !reflect.DeepEqual(checkpoint.Publications, wantPublications) { + t.Fatalf("checkpoint publications = %+v, want %+v", checkpoint.Publications, wantPublications) + } +} + +func TestOrchestratorResumeReconcilesBeforeContinuing(t *testing.T) { + t.Parallel() + + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + outputDir := t.TempDir() + interrupted := errors.New("rollout observation interrupted") + harness.cluster.waitForSignerError = interrupted + + options := RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: outputDir, + } + result, err := harness.orchestrator.Run(context.Background(), options) + if !errors.Is(err, interrupted) { + t.Fatalf("initial Run() error = %v, want %v", err, interrupted) + } + if result.Phase != PhaseCombinedJWKSPublished || result.Complete { + t.Fatalf("initial Run() result = %+v, want interruption after combined publication", result) + } + if !reflect.DeepEqual(harness.publisher.publications, []string{"combined"}) { + t.Fatalf("initial provider publications = %v, want only combined", harness.publisher.publications) + } + + harness.events = nil + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if err != nil { + t.Fatalf("resumed Run() returned unexpected error: %v", err) + } + if result.Phase != PhaseComplete || !result.Complete { + t.Fatalf("resumed Run() result = %+v, want complete phase", result) + } + + wantPrefix := []string{ + "cluster.observe-rotation-guard", + "target.resolve", + "publisher.check-access", + "cluster.observe-signer-reference", + "cluster.observe-public-signer-bundle", + "cluster.observe-signer-reference", + "cluster.reconcile:combined-jwks-published", + "publisher.read:combined", + "cluster.observe-rotation-guard", + "cluster.wait-for-signer-rollout", + } + if len(harness.events) < len(wantPrefix) || !reflect.DeepEqual(harness.events[:len(wantPrefix)], wantPrefix) { + t.Fatalf("resumed external effect prefix = %v, want %v", harness.events, wantPrefix) + } + if !reflect.DeepEqual(harness.publisher.publications, []string{"combined", "new-only"}) { + t.Fatalf("provider publications across resume = %v, want no repeated combined publication", harness.publisher.publications) + } + if len(harness.cluster.expectations) != 2 { + t.Fatalf("reconciliation calls = %d, want 2", len(harness.cluster.expectations)) + } + firstExpectation := harness.cluster.expectations[0] + if firstExpectation.Phase != PhaseCombinedJWKSPublished { + t.Fatalf("first reconciliation phase = %q, want %q", firstExpectation.Phase, PhaseCombinedJWKSPublished) + } + if firstExpectation.ClusterIdentity != harness.clusterIdentity { + t.Fatalf("reconciled cluster identity = %q, want %q", firstExpectation.ClusterIdentity, harness.clusterIdentity) + } + if len(firstExpectation.PreRotationSignerBaseline.Entries) != 1 || firstExpectation.PreRotationSignerBaseline.Entries[0].KeyID != harness.initialKeyID { + t.Fatalf("reconciled pre-rotation signer baseline = %#v, want key %q", firstExpectation.PreRotationSignerBaseline, harness.initialKeyID) + } + if firstExpectation.ReplacementSigner == nil || firstExpectation.ReplacementSigner.Entry.KeyID != harness.replacementKeyID { + t.Fatalf("reconciled replacement signer = %#v, want key %q", firstExpectation.ReplacementSigner, harness.replacementKeyID) + } + if firstExpectation.RebootIntent != nil { + t.Fatalf("reconciliation before reboot intent unexpectedly received %+v", firstExpectation.RebootIntent) + } +} + +func requireOrchestratorPause(t *testing.T, err error, phase Phase, reason PauseReason, artifact string) *PauseError { + t.Helper() + var pause *PauseError + if !errors.As(err, &pause) { + t.Fatalf("Run() error = %v, want PauseError", err) + } + if pause.Phase != phase || pause.Reason != reason || pause.Artifact != artifact { + t.Fatalf("PauseError = %+v, want phase %q, reason %q, artifact %q", pause, phase, reason, artifact) + } + return pause +} + +type orchestratorTestHarness struct { + events []string + orchestrator Orchestrator + cluster *fakeClusterRotation + publisher *fakeConditionalJWKSBackend + initialReference SignerObjectReference + currentJWKS []byte + initialKeyID string + replacementKeyID string + clusterIdentity string + combinedSHA256 string + newOnlySHA256 string +} + +func newOrchestratorTestHarness(t *testing.T, mode PublicationMode) *orchestratorTestHarness { + t.Helper() + + initialPublicKey := testPublicKeyPEM(t) + replacementPublicKey := testPublicKeyPEM(t) + currentJWKS := encodedJWKSForTest(t, initialPublicKey) + prepared, err := PrepareJWKSArtifacts(currentJWKS, replacementPublicKey) + if err != nil { + t.Fatalf("PrepareJWKSArtifacts() returned unexpected error: %v", err) + } + + initialReference := SignerObjectReference{ + UID: "initial-signer-uid", + ResourceVersion: "100", + } + replacementReference := SignerObjectReference{ + UID: "replacement-signer-uid", + ResourceVersion: "200", + } + initialBundle := PublicSignerBundleObservation{ + ConfigMapUID: "signer-configmap-uid", + ConfigMapResourceVersion: "1000", + Signers: []PublicSignerObservation{{ + Name: "service-account-001.pub", + PublicKeyPEM: initialPublicKey, + }}, + } + replacement := PublicSignerObservation{ + Name: "service-account-002.pub", + PublicKeyPEM: replacementPublicKey, + } + initialState, err := normalizePublicSignerBundle(initialBundle) + if err != nil { + t.Fatalf("validate initial signer fixture: %v", err) + } + replacementState, err := normalizePublicSignerBundle(PublicSignerBundleObservation{ + ConfigMapUID: initialBundle.ConfigMapUID, + ConfigMapResourceVersion: "1001", + Signers: append(clonePublicSignerObservationsForTest(initialBundle.Signers), replacement), + }) + if err != nil { + t.Fatalf("validate replacement signer fixture: %v", err) + } + initialKeyID := initialState.Baseline.Entries[0].KeyID + replacementKeyID := replacementState.Baseline.Entries[1].KeyID + + harness := &orchestratorTestHarness{ + initialReference: initialReference, + currentJWKS: append([]byte(nil), currentJWKS...), + initialKeyID: initialKeyID, + replacementKeyID: replacementKeyID, + clusterIdentity: "cluster-test-identity", + combinedSHA256: prepared.Combined.SHA256, + newOnlySHA256: prepared.New.SHA256, + } + harness.cluster = &fakeClusterRotation{ + events: &harness.events, + clusterIdentity: harness.clusterIdentity, + nextReference: cloneSignerReferenceForTest(&initialReference), + nextBundle: clonePublicSignerBundleForTest(initialBundle), + replacementReference: replacementReference, + replacement: clonePublicSignerObservationForTest(replacement), + rebootIntent: RebootIntent{ + ID: "rotation-test-intent", + Targets: []string{"worker"}, + Baselines: []NodeRebootBaseline{ + {Target: "worker", Node: "worker-0", BootID: "initial-boot-id"}, + }, + }, + rebootStatus: RebootNotStarted, + } + target := &fakeTargetResolver{ + events: &harness.events, + target: "aws://test-issuer/jwks", + } + harness.publisher = &fakeConditionalJWKSBackend{ + events: &harness.events, + target: target.target, + data: append([]byte(nil), currentJWKS...), + current: append([]byte(nil), currentJWKS...), + combined: append([]byte(nil), prepared.Combined.Data...), + newOnly: append([]byte(nil), prepared.New.Data...), + revision: 1, + } + harness.orchestrator = Orchestrator{ + Cluster: harness.cluster, + Target: target, + } + if mode == PublicationModeDirect { + harness.orchestrator.Publisher = harness.publisher + } + return harness +} + +type fakeTargetResolver struct { + events *[]string + target string +} + +func (f *fakeTargetResolver) ResolveTarget(context.Context) (string, error) { + *f.events = append(*f.events, "target.resolve") + return f.target, nil +} + +type fakeConditionalJWKSBackend struct { + events *[]string + target string + data []byte + current []byte + combined []byte + newOnly []byte + revision int + publications []string +} + +func (f *fakeConditionalJWKSBackend) CheckAccess(_ context.Context, target string) error { + *f.events = append(*f.events, "publisher.check-access") + if target != f.target { + return fmt.Errorf("target = %q, want %q", target, f.target) + } + return nil +} + +func (f *fakeConditionalJWKSBackend) ReadJWKS(_ context.Context, target string) (VersionedJWKS, error) { + label := f.dataLabel(f.data) + *f.events = append(*f.events, "publisher.read:"+label) + if target != f.target { + return VersionedJWKS{}, fmt.Errorf("target = %q, want %q", target, f.target) + } + return VersionedJWKS{ + Data: append([]byte(nil), f.data...), + Revision: fmt.Sprintf("revision-%d", f.revision), + }, nil +} + +func (f *fakeConditionalJWKSBackend) PublishIfVersion(_ context.Context, target, revision string, data []byte) (EffectOutcome, error) { + label := f.dataLabel(data) + *f.events = append(*f.events, "publisher.publish:"+label) + if target != f.target { + return EffectNotApplied, fmt.Errorf("target = %q, want %q", target, f.target) + } + wantRevision := fmt.Sprintf("revision-%d", f.revision) + if revision != wantRevision { + return EffectNotApplied, fmt.Errorf("revision = %q, want %q", revision, wantRevision) + } + if label != "combined" && label != "new-only" { + return EffectNotApplied, fmt.Errorf("unexpected publication payload") + } + f.data = append([]byte(nil), data...) + f.revision++ + f.publications = append(f.publications, label) + return EffectSubmitted, nil +} + +func (f *fakeConditionalJWKSBackend) dataLabel(data []byte) string { + switch { + case bytes.Equal(data, f.current): + return "current" + case bytes.Equal(data, f.combined): + return "combined" + case bytes.Equal(data, f.newOnly): + return "new-only" + default: + return "unknown" + } +} + +type fakeClusterRotation struct { + events *[]string + clusterIdentity string + nextReference *SignerObjectReference + nextBundle PublicSignerBundleObservation + replacementReference SignerObjectReference + replacement PublicSignerObservation + rebootIntent RebootIntent + canonicalReboot *RebootIntent + rebootStatus RebootStatus + waitForSignerError error + replacementRequests int + rebootRequests int + bundleObservations int + applyReplacementAt int + requestedReference SignerObjectReference + waitedReference SignerObjectReference + rolloutKeyID string + rebootKeyID string + expectations []ClusterExpectation + guardReference *RotationGuardReference + completedGuards map[string]struct{} + guardAcquireCalls int + guardAcquireMutations int + guardReleaseCalls int + guardReleaseMutations int +} + +func (f *fakeClusterRotation) Preflight(context.Context) (ClusterPreflight, error) { + *f.events = append(*f.events, "cluster.preflight") + return ClusterPreflight{ + ClusterIdentity: f.clusterIdentity, + }, nil +} + +func (f *fakeClusterRotation) ObserveRotationGuard(_ context.Context, reference RotationGuardReference) (RotationGuardObservation, error) { + *f.events = append(*f.events, "cluster.observe-rotation-guard") + if _, completed := f.completedGuards[reference.OperationID]; completed { + return RotationGuardObservation{Status: RotationGuardCompleted, OperationID: reference.OperationID}, nil + } + if f.guardReference == nil { + return RotationGuardObservation{Status: RotationGuardNotFound}, nil + } + if *f.guardReference == reference { + return RotationGuardObservation{Status: RotationGuardHeld, OperationID: reference.OperationID}, nil + } + return RotationGuardObservation{Status: RotationGuardOwnedByOther, OperationID: f.guardReference.OperationID}, nil +} + +func (f *fakeClusterRotation) AcquireRotationGuard(_ context.Context, reference RotationGuardReference) (EffectOutcome, error) { + *f.events = append(*f.events, "cluster.acquire-rotation-guard") + f.guardAcquireCalls++ + if f.guardReference == nil { + stored := reference + f.guardReference = &stored + f.guardAcquireMutations++ + return EffectSubmitted, nil + } + if *f.guardReference == reference { + return EffectSubmitted, nil + } + return EffectNotApplied, nil +} + +func (f *fakeClusterRotation) ReleaseRotationGuard(_ context.Context, reference RotationGuardReference) (EffectOutcome, error) { + *f.events = append(*f.events, "cluster.release-rotation-guard") + f.guardReleaseCalls++ + if _, completed := f.completedGuards[reference.OperationID]; completed { + return EffectSubmitted, nil + } + if f.guardReference == nil || *f.guardReference != reference { + return EffectNotApplied, nil + } + f.guardReference = nil + if f.completedGuards == nil { + f.completedGuards = make(map[string]struct{}) + } + f.completedGuards[reference.OperationID] = struct{}{} + f.guardReleaseMutations++ + return EffectSubmitted, nil +} + +func (f *fakeClusterRotation) Reconcile(_ context.Context, guard RotationGuardReference, expectation ClusterExpectation) error { + *f.events = append(*f.events, "cluster.reconcile:"+string(expectation.Phase)) + if err := f.requireHeldRotationGuard(guard); err != nil { + return err + } + if expectation.RotationGuard != guard { + return fmt.Errorf("reconciliation guard = %#v, want %#v", guard, expectation.RotationGuard) + } + f.expectations = append(f.expectations, cloneClusterExpectationForTest(expectation)) + return nil +} + +func (f *fakeClusterRotation) ObserveSignerReference(_ context.Context, guard *RotationGuardReference) (*SignerObjectReference, error) { + *f.events = append(*f.events, "cluster.observe-signer-reference") + if guard != nil { + if err := f.requireHeldRotationGuard(*guard); err != nil { + return nil, err + } + } + return cloneSignerReferenceForTest(f.nextReference), nil +} + +func (f *fakeClusterRotation) ObservePublicSignerBundle(_ context.Context, guard *RotationGuardReference) (PublicSignerBundleObservation, error) { + *f.events = append(*f.events, "cluster.observe-public-signer-bundle") + if guard != nil { + if err := f.requireHeldRotationGuard(*guard); err != nil { + return PublicSignerBundleObservation{}, err + } + } + f.bundleObservations++ + if f.applyReplacementAt == f.bundleObservations { + f.applyReplacementForTest() + } + return clonePublicSignerBundleForTest(f.nextBundle), nil +} + +func (f *fakeClusterRotation) RequestReplacement(_ context.Context, guard RotationGuardReference, reference SignerObjectReference) (EffectOutcome, error) { + *f.events = append(*f.events, "cluster.request-replacement") + if err := f.requireHeldRotationGuard(guard); err != nil { + return EffectNotApplied, err + } + f.replacementRequests++ + f.requestedReference = reference + f.applyReplacementForTest() + return EffectSubmitted, nil +} + +func (f *fakeClusterRotation) applyReplacementForTest() { + f.nextReference = cloneSignerReferenceForTest(&f.replacementReference) + f.nextBundle.ConfigMapResourceVersion = "1001" + for _, signer := range f.nextBundle.Signers { + if signer.Name == f.replacement.Name { + return + } + } + f.nextBundle.Signers = append(f.nextBundle.Signers, clonePublicSignerObservationForTest(f.replacement)) +} + +func (f *fakeClusterRotation) WaitForReplacement(_ context.Context, guard RotationGuardReference, reference SignerObjectReference) error { + *f.events = append(*f.events, "cluster.wait-for-replacement") + if err := f.requireHeldRotationGuard(guard); err != nil { + return err + } + f.waitedReference = reference + return nil +} + +func (f *fakeClusterRotation) WaitForSignerRollout(_ context.Context, guard RotationGuardReference, replacementKeyID string) error { + *f.events = append(*f.events, "cluster.wait-for-signer-rollout") + if err := f.requireHeldRotationGuard(guard); err != nil { + return err + } + f.rolloutKeyID = replacementKeyID + if f.waitForSignerError != nil { + err := f.waitForSignerError + f.waitForSignerError = nil + return err + } + return nil +} + +func (f *fakeClusterRotation) PrepareReboot(_ context.Context, guard RotationGuardReference, replacementKeyID string) (RebootPlan, error) { + *f.events = append(*f.events, "cluster.prepare-reboot") + if err := f.requireHeldRotationGuard(guard); err != nil { + return RebootPlan{}, err + } + f.rebootKeyID = replacementKeyID + intent := cloneRebootIntent(f.rebootIntent) + return RebootPlan{Targets: intent.Targets, Baselines: intent.Baselines}, nil +} + +func (f *fakeClusterRotation) ObserveReboot(_ context.Context, guard RotationGuardReference, operationID string) (RebootObservation, error) { + *f.events = append(*f.events, "cluster.observe-reboot:"+string(f.rebootStatus)) + if err := f.requireHeldRotationGuard(guard); err != nil { + return RebootObservation{}, err + } + if f.rebootStatus == RebootNotStarted { + return RebootObservation{Status: RebootNotStarted}, nil + } + intent := f.rebootIntent + if f.canonicalReboot != nil { + intent = *f.canonicalReboot + } else { + intent.ID = operationID + } + intent = cloneRebootIntent(intent) + return RebootObservation{Status: f.rebootStatus, CanonicalIntent: &intent}, nil +} + +func (f *fakeClusterRotation) RequestReboot(_ context.Context, guard RotationGuardReference, intent RebootIntent) (EffectOutcome, error) { + *f.events = append(*f.events, "cluster.request-reboot") + if err := f.requireHeldRotationGuard(guard); err != nil { + return EffectNotApplied, err + } + f.rebootRequests++ + if f.canonicalReboot == nil { + canonical := cloneRebootIntent(intent) + f.canonicalReboot = &canonical + } + f.rebootStatus = RebootInProgress + return EffectSubmitted, nil +} + +func (f *fakeClusterRotation) WaitForReboot(_ context.Context, guard RotationGuardReference, _ RebootIntent) error { + *f.events = append(*f.events, "cluster.wait-for-reboot") + if err := f.requireHeldRotationGuard(guard); err != nil { + return err + } + f.rebootStatus = RebootComplete + return nil +} + +func (f *fakeClusterRotation) WaitForPostRebootStable(_ context.Context, guard RotationGuardReference, _ RebootIntent) error { + *f.events = append(*f.events, "cluster.wait-for-post-reboot-stable") + if err := f.requireHeldRotationGuard(guard); err != nil { + return err + } + return nil +} + +func (f *fakeClusterRotation) requireHeldRotationGuard(guard RotationGuardReference) error { + if f.guardReference == nil { + return fmt.Errorf("operation does not hold a signer-rotation guard") + } + if *f.guardReference != guard { + return fmt.Errorf("operation guard = %#v, want held guard %#v", guard, *f.guardReference) + } + return nil +} + +func (f *fakeClusterRotation) reconciledPhases() []Phase { + phases := make([]Phase, 0, len(f.expectations)) + for _, expectation := range f.expectations { + phases = append(phases, expectation.Phase) + } + return phases +} + +func cloneSignerReferenceForTest(reference *SignerObjectReference) *SignerObjectReference { + if reference == nil { + return nil + } + clone := *reference + return &clone +} + +func clonePublicSignerObservationForTest(observation PublicSignerObservation) PublicSignerObservation { + observation.PublicKeyPEM = append([]byte(nil), observation.PublicKeyPEM...) + return observation +} + +func clonePublicSignerObservationsForTest(observations []PublicSignerObservation) []PublicSignerObservation { + clones := make([]PublicSignerObservation, len(observations)) + for index, observation := range observations { + clones[index] = clonePublicSignerObservationForTest(observation) + } + return clones +} + +func clonePublicSignerBundleForTest(bundle PublicSignerBundleObservation) PublicSignerBundleObservation { + bundle.Signers = clonePublicSignerObservationsForTest(bundle.Signers) + return bundle +} + +func cloneClusterExpectationForTest(expectation ClusterExpectation) ClusterExpectation { + expectation.PreRotationSignerBaseline = clonePublicSignerBaseline(expectation.PreRotationSignerBaseline) + if expectation.ReplacementSigner != nil { + evidence := cloneReplacementSignerEvidence(*expectation.ReplacementSigner) + expectation.ReplacementSigner = &evidence + } + if expectation.RebootIntent != nil { + intent := cloneRebootIntent(*expectation.RebootIntent) + expectation.RebootIntent = &intent + } + return expectation +} diff --git a/pkg/cmd/provisioning/rotation/orchestrator_types.go b/pkg/cmd/provisioning/rotation/orchestrator_types.go new file mode 100644 index 0000000000..2903bf08e5 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/orchestrator_types.go @@ -0,0 +1,282 @@ +package rotation + +import ( + "context" + "fmt" +) + +// EffectOutcome describes what an adapter knows about an external mutation. +// Unknown outcomes must be reconciled through observation before an adapter is +// asked to repeat the mutation. +type EffectOutcome string + +const ( + EffectSubmitted EffectOutcome = "submitted" + EffectNotApplied EffectOutcome = "not-applied" + EffectUnknown EffectOutcome = "unknown" +) + +// PublicSignerObservation is one named public value from the cumulative signer +// ConfigMap. Entry names do not identify an active or next signer. +type PublicSignerObservation struct { + Name string + PublicKeyPEM []byte +} + +// PublicSignerBundleObservation is a complete read of the public-only signer +// ConfigMap. Implementations must return every data entry without selecting or +// reordering a presumed active signer. +type PublicSignerBundleObservation struct { + ConfigMapUID string + ConfigMapResourceVersion string + Signers []PublicSignerObservation +} + +// ClusterPreflight is returned only after the adapter has confirmed cluster +// stability and the permissions needed by the shared workflow. +type ClusterPreflight struct { + ClusterIdentity string +} + +// ClusterExpectation describes the non-secret evidence an adapter must +// reconcile before a resumed workflow trusts any prior checkpoint. +type ClusterExpectation struct { + Phase Phase + ClusterIdentity string + RotationGuard RotationGuardReference + PreRotationSignerBaseline PublicSignerBaseline + PreRotationSignerRef SignerObjectReference + ReplacementSigner *ReplacementSignerEvidence + RebootIntent *RebootIntent +} + +// RotationGuardStatus is the observable state of one exact signer-rotation +// operation in its cluster-global guard scope. +type RotationGuardStatus string + +const ( + RotationGuardNotFound RotationGuardStatus = "not-found" + RotationGuardHeld RotationGuardStatus = "held" + RotationGuardCompleted RotationGuardStatus = "completed" + RotationGuardOwnedByOther RotationGuardStatus = "owned-by-other" +) + +// RotationGuardObservation is a read-only lookup of one exact operation. For +// Held and Completed, OperationID is the requested operation. For +// OwnedByOther, it is the conflicting active operation. A Completed result is +// a durable per-operation terminal record and remains observable after a later +// operation acquires the same cluster-global scope. +type RotationGuardObservation struct { + Status RotationGuardStatus + OperationID string +} + +// RebootStatus is the observable state of a cluster-durable reboot record. +type RebootStatus string + +const ( + RebootNotStarted RebootStatus = "not-started" + RebootInProgress RebootStatus = "in-progress" + RebootComplete RebootStatus = "complete" +) + +// RebootPlan is the adapter-observed set of machine config pool targets and +// node boot-ID baselines. The shared engine canonicalizes this plan and assigns +// the deterministic RebootIntent ID before it becomes durable. +type RebootPlan struct { + Targets []string + Baselines []NodeRebootBaseline +} + +// RebootObservation is a read-only lookup of one deterministic reboot +// operation ID. CanonicalIntent is nil only when Status is RebootNotStarted. +// Otherwise it is the immutable intent stored by the first RequestReboot call. +type RebootObservation struct { + Status RebootStatus + CanonicalIntent *RebootIntent +} + +// ClusterRotation supplies provider-independent cluster observations and +// actions. Disruptive methods receive immutable checkpoint evidence so their +// implementations can use API preconditions and idempotent operation IDs. +// +// Preflight, Reconcile, every Observe* and Wait* method, and PrepareReboot are +// read-only. AcquireRotationGuard, ReleaseRotationGuard, RequestReplacement, +// and RequestReboot are the only mutation entry points. Reconcile validates +// identity and all claims that are durable at the +// supplied phase. It must tolerate the in-between observable state of an effect +// whose checkpoint was interrupted: at PhaseCurrentJWKSRead the original signer +// may already be absent or replaced, and at PhaseRebootIntentRecorded the exact +// persisted reboot may be not started, in progress, or complete. +// +// RequestReboot is the sole reboot mutation. It must atomically create the +// cluster-durable canonical record and request the reboot when the ID is new. +// For an existing ID it must preserve the first canonical intent and must not +// request another reboot. The canonical record must remain observable while a +// checkpoint for the operation can be resumed. RebootNotStarted means that no +// queued, active, or completed operation with that ID is observable. +// ObserveSignerReference must request only the meta.k8s.io/v1 +// PartialObjectMetadata representation of the next-signer Secret and must fail +// closed rather than accepting a full Secret fallback. A nil reference means +// only that the object was not found. ObservePublicSignerBundle must read only +// the public openshift-kube-apiserver/bound-sa-token-signing-certs ConfigMap. +// Shared orchestration surrounds each bundle read with two metadata-reference +// reads and accepts the observation only when those references match. +// +// ObserveRotationGuard looks up the exact operation within the cluster-global +// signer-rotation scope. AcquireRotationGuard must atomically claim a free +// scope, converge when the same operation already owns it, and report a +// different active operation as OwnedByOther. ReleaseRotationGuard must only +// release the exact active operation and must atomically leave a durable +// Completed record for it. Unknown mutation outcomes are reconciled by another +// observation before any retry. +// +// The public ConfigMap has no field that cryptographically binds an entry to a +// particular next-signer Secret UID. Every method after preflight evidence +// capture therefore receives the exact immutable guard reference. The two +// signer observation methods accept a nil reference only for the initial +// read-only ref-bundle-ref snapshot needed to derive that reference; every +// later call must supply it. Implementations must fail closed if that operation +// no longer owns the guard. Multiple +// workspaces may reconcile the same guarded operation; different preflight +// evidence derives a different operation and must conflict in the shared scope. +// +// RequestReplacement must issue a UID/resource-version-preconditioned delete +// and be idempotent for the same SignerObjectReference. EffectSubmitted alone +// does not advance the checkpoint; absence or a changed Secret UID must be +// observed first. WaitForReplacement is read-only and only waits for cluster +// progress; shared orchestration re-observes and verifies the exact bundle +// delta before trusting a replacement. +type ClusterRotation interface { + Preflight(context.Context) (ClusterPreflight, error) + ObserveRotationGuard(context.Context, RotationGuardReference) (RotationGuardObservation, error) + AcquireRotationGuard(context.Context, RotationGuardReference) (EffectOutcome, error) + ReleaseRotationGuard(context.Context, RotationGuardReference) (EffectOutcome, error) + Reconcile(context.Context, RotationGuardReference, ClusterExpectation) error + ObserveSignerReference(context.Context, *RotationGuardReference) (*SignerObjectReference, error) + ObservePublicSignerBundle(context.Context, *RotationGuardReference) (PublicSignerBundleObservation, error) + RequestReplacement(context.Context, RotationGuardReference, SignerObjectReference) (EffectOutcome, error) + WaitForReplacement(context.Context, RotationGuardReference, SignerObjectReference) error + WaitForSignerRollout(context.Context, RotationGuardReference, string) error + PrepareReboot(context.Context, RotationGuardReference, string) (RebootPlan, error) + ObserveReboot(context.Context, RotationGuardReference, string) (RebootObservation, error) + RequestReboot(context.Context, RotationGuardReference, RebootIntent) (EffectOutcome, error) + WaitForReboot(context.Context, RotationGuardReference, RebootIntent) error + WaitForPostRebootStable(context.Context, RotationGuardReference, RebootIntent) error +} + +// TargetResolver returns a stable, non-secret identity for the issuer JWKS +// target. The same identity must be returned throughout a resumed operation. +type TargetResolver interface { + ResolveTarget(context.Context) (string, error) +} + +// VersionedJWKS is an exact provider read plus the opaque revision required by +// a subsequent conditional write. +type VersionedJWKS struct { + Data []byte + Revision string +} + +// ConditionalJWKSBackend implements direct publication without blind +// overwrites. PublishIfVersion must submit no write when the revision no longer +// matches. +type ConditionalJWKSBackend interface { + CheckAccess(context.Context, string) error + ReadJWKS(context.Context, string) (VersionedJWKS, error) + PublishIfVersion(context.Context, string, string, []byte) (EffectOutcome, error) +} + +// ManualAcknowledgement confirms that the caller applied one exact public +// artifact. It is valid for only the publication phase named here. +type ManualAcknowledgement struct { + Phase Phase + Artifact string + SHA256 string +} + +// ManualInput supplies information that cannot be observed by the engine in +// manual publication mode. Only one acknowledgement can be consumed per run. +type ManualInput struct { + CurrentJWKS []byte + Acknowledgement *ManualAcknowledgement +} + +// RunOptions identifies one new or resumed rotation operation. +type RunOptions struct { + Provider Provider + PublicationMode PublicationMode + OutputDir string + Resume bool + Manual ManualInput +} + +// RunResult reports the last durable phase reached by Run. +type RunResult struct { + Phase Phase + Complete bool +} + +// PauseReason identifies an expected manual hand-off rather than a failure. +type PauseReason string + +const ( + PauseForCurrentJWKS PauseReason = "current-jwks-required" + PauseForPublication PauseReason = "publication-acknowledgement-required" +) + +// PauseError tells a caller which public input or publication confirmation is +// required before resuming. +type PauseError struct { + Phase Phase + Reason PauseReason + Artifact string + Path string + SHA256 string +} + +func (e *PauseError) Error() string { + if e == nil { + return "rotation is paused" + } + if e.Reason == PauseForPublication { + return fmt.Sprintf("rotation paused at phase %q: apply artifact %q with SHA-256 %s and resume with an acknowledgement", e.Phase, e.Artifact, e.SHA256) + } + return fmt.Sprintf("rotation paused at phase %q: supply the current JWKS and resume", e.Phase) +} + +// OutcomeUnknownError reports an external mutation whose final state could not +// be proven. A later resume must reconcile observation before retrying it. +type OutcomeUnknownError struct { + Phase Phase + Operation string + Cause error +} + +func (e *OutcomeUnknownError) Error() string { + if e == nil { + return "rotation external operation outcome is unknown" + } + return fmt.Sprintf("rotation operation %q at phase %q has an unknown outcome; inspect observable state and resume", e.Operation, e.Phase) +} + +func (e *OutcomeUnknownError) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} + +// ConflictError reports observable state that is neither the expected +// predecessor nor the exact desired result. +type ConflictError struct { + Phase Phase + Reason string +} + +func (e *ConflictError) Error() string { + if e == nil { + return "rotation state conflict" + } + return fmt.Sprintf("rotation state conflict at phase %q: %s", e.Phase, e.Reason) +} diff --git a/pkg/cmd/provisioning/rotation/publication_overlap_test.go b/pkg/cmd/provisioning/rotation/publication_overlap_test.go new file mode 100644 index 0000000000..6892b9dd25 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/publication_overlap_test.go @@ -0,0 +1,54 @@ +package rotation + +import ( + "context" + "errors" + "strings" + "testing" +) + +func TestOrchestratorRechecksCombinedJWKSBeforePlanningReboot(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + cluster := &driftAfterSignerRolloutCluster{ + fakeClusterRotation: harness.cluster, + publisher: harness.publisher, + driftedJWKS: encodedJWKSForTest(t, testPublicKeyPEM(t)), + } + harness.orchestrator.Cluster = cluster + + result, err := harness.orchestrator.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: t.TempDir(), + }) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() error = %v, want ConflictError", err) + } + if result.Phase != PhaseCombinedJWKSPublished || result.Complete { + t.Fatalf("Run() result = %#v, want failure before signer-rollout-stable checkpoint", result) + } + if harness.cluster.rebootKeyID != "" || harness.cluster.rebootRequests != 0 { + t.Fatalf("reboot planning key ID = %q, requests = %d; want no reboot activity", harness.cluster.rebootKeyID, harness.cluster.rebootRequests) + } + for _, event := range harness.events { + if event == "cluster.prepare-reboot" || event == "cluster.request-reboot" || strings.HasPrefix(event, "cluster.observe-reboot:") { + t.Fatalf("unexpected reboot activity after provider drift: %q", event) + } + } +} + +type driftAfterSignerRolloutCluster struct { + *fakeClusterRotation + publisher *fakeConditionalJWKSBackend + driftedJWKS []byte +} + +func (cluster *driftAfterSignerRolloutCluster) WaitForSignerRollout(ctx context.Context, guard RotationGuardReference, replacementKeyID string) error { + if err := cluster.fakeClusterRotation.WaitForSignerRollout(ctx, guard, replacementKeyID); err != nil { + return err + } + cluster.publisher.data = append([]byte(nil), cluster.driftedJWKS...) + cluster.publisher.revision++ + return nil +} diff --git a/pkg/cmd/provisioning/rotation/reboot_intent.go b/pkg/cmd/provisioning/rotation/reboot_intent.go new file mode 100644 index 0000000000..296162ce26 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/reboot_intent.go @@ -0,0 +1,51 @@ +package rotation + +import ( + "crypto/sha256" + "fmt" + "slices" + "strings" +) + +func buildRebootIntent(clusterIdentity, replacementKeyID string, plan RebootPlan) (RebootIntent, error) { + targets := append([]string(nil), plan.Targets...) + slices.Sort(targets) + baselines := append([]NodeRebootBaseline(nil), plan.Baselines...) + slices.SortFunc(baselines, func(left, right NodeRebootBaseline) int { + if comparison := strings.Compare(left.Target, right.Target); comparison != 0 { + return comparison + } + if comparison := strings.Compare(left.Node, right.Node); comparison != 0 { + return comparison + } + return strings.Compare(left.BootID, right.BootID) + }) + + intentID, err := rebootIntentID(clusterIdentity, replacementKeyID) + if err != nil { + return RebootIntent{}, err + } + intent := RebootIntent{ID: intentID, Targets: targets, Baselines: baselines} + if err := validateRebootIntent(intent); err != nil { + return RebootIntent{}, err + } + return intent, nil +} + +func rebootIntentID(clusterIdentity, replacementKeyID string) (string, error) { + if err := validateOpaqueCheckpointValue("cluster identity", clusterIdentity); err != nil { + return "", err + } + if err := validateDerivedKeyID(replacementKeyID); err != nil { + return "", fmt.Errorf("invalid replacement key identity: %w", err) + } + + digest := sha256.New() + writePart := func(value string) { + _, _ = fmt.Fprintf(digest, "%d:%s", len(value), value) + } + writePart("signer-rotation-reboot-v1") + writePart(clusterIdentity) + writePart(replacementKeyID) + return fmt.Sprintf("signer-rotation-%x", digest.Sum(nil)), nil +} diff --git a/pkg/cmd/provisioning/rotation/rotation_guard.go b/pkg/cmd/provisioning/rotation/rotation_guard.go new file mode 100644 index 0000000000..bf903c1a1d --- /dev/null +++ b/pkg/cmd/provisioning/rotation/rotation_guard.go @@ -0,0 +1,84 @@ +package rotation + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" +) + +const ( + rotationGuardScopeDomain = "signer-rotation-guard-scope-v1" + rotationGuardOperationDomain = "signer-rotation-guard-operation-v1" +) + +type rotationGuardScopePayload struct { + Domain string `json:"domain"` + ClusterIdentity string `json:"clusterIdentity"` +} + +type rotationGuardOperationPayload struct { + Domain string `json:"domain"` + ClusterIdentity string `json:"clusterIdentity"` + Provider Provider `json:"provider"` + TargetIdentity string `json:"targetIdentity"` + PreRotationSignerBaseline PublicSignerBaseline `json:"preRotationSignerBaseline"` + PreRotationSignerRef SignerObjectReference `json:"preRotationSignerRef"` +} + +func deriveRotationGuardReference(clusterIdentity string, provider Provider, targetIdentity string, baseline PublicSignerBaseline, signerRef SignerObjectReference) (RotationGuardReference, error) { + if err := validateOpaqueCheckpointValue("cluster identity", clusterIdentity); err != nil { + return RotationGuardReference{}, err + } + if !isSupportedProvider(provider) { + return RotationGuardReference{}, fmt.Errorf("unsupported rotation provider %q", provider) + } + if err := validateOpaqueCheckpointValue("target identity", targetIdentity); err != nil { + return RotationGuardReference{}, err + } + if err := validatePublicSignerBaseline(baseline); err != nil { + return RotationGuardReference{}, fmt.Errorf("invalid pre-rotation public signer baseline: %w", err) + } + if err := validateSignerObjectReference(signerRef); err != nil { + return RotationGuardReference{}, fmt.Errorf("invalid pre-rotation signer object reference: %w", err) + } + + scopeID, err := hashRotationGuardPayload(rotationGuardScopePayload{ + Domain: rotationGuardScopeDomain, + ClusterIdentity: clusterIdentity, + }) + if err != nil { + return RotationGuardReference{}, fmt.Errorf("encode rotation guard scope: %w", err) + } + operationID, err := hashRotationGuardPayload(rotationGuardOperationPayload{ + Domain: rotationGuardOperationDomain, + ClusterIdentity: clusterIdentity, + Provider: provider, + TargetIdentity: targetIdentity, + PreRotationSignerBaseline: clonePublicSignerBaseline(baseline), + PreRotationSignerRef: signerRef, + }) + if err != nil { + return RotationGuardReference{}, fmt.Errorf("encode rotation guard operation: %w", err) + } + return RotationGuardReference{ScopeID: scopeID, OperationID: operationID}, nil +} + +func hashRotationGuardPayload(payload any) (string, error) { + encoded, err := json.Marshal(payload) + if err != nil { + return "", err + } + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]), nil +} + +func validateRotationGuardReference(reference RotationGuardReference) error { + if err := validateSHA256(reference.ScopeID); err != nil { + return fmt.Errorf("rotation guard scope ID: %w", err) + } + if err := validateSHA256(reference.OperationID); err != nil { + return fmt.Errorf("rotation guard operation ID: %w", err) + } + return nil +} diff --git a/pkg/cmd/provisioning/rotation/rotation_guard_orchestrator.go b/pkg/cmd/provisioning/rotation/rotation_guard_orchestrator.go new file mode 100644 index 0000000000..d7eb66158b --- /dev/null +++ b/pkg/cmd/provisioning/rotation/rotation_guard_orchestrator.go @@ -0,0 +1,234 @@ +package rotation + +import ( + "context" + "errors" + "fmt" +) + +func (o Orchestrator) observeRotationGuard(ctx context.Context, reference RotationGuardReference) (RotationGuardObservation, error) { + observation, err := o.Cluster.ObserveRotationGuard(ctx, reference) + if err != nil { + return RotationGuardObservation{}, err + } + if err := validateRotationGuardObservation(reference, observation); err != nil { + return RotationGuardObservation{}, err + } + return observation, nil +} + +func validateRotationGuardObservation(reference RotationGuardReference, observation RotationGuardObservation) error { + if err := validateRotationGuardReference(reference); err != nil { + return err + } + switch observation.Status { + case RotationGuardNotFound: + if observation.OperationID != "" { + return fmt.Errorf("adapter returned operation ID %q for rotation guard status %q", observation.OperationID, observation.Status) + } + case RotationGuardHeld, RotationGuardCompleted: + if observation.OperationID != reference.OperationID { + return fmt.Errorf("adapter returned rotation guard operation %q for requested operation %q", observation.OperationID, reference.OperationID) + } + case RotationGuardOwnedByOther: + if err := validateSHA256(observation.OperationID); err != nil { + return fmt.Errorf("adapter returned invalid conflicting rotation guard operation ID: %w", err) + } + if observation.OperationID == reference.OperationID { + return fmt.Errorf("adapter reported requested rotation guard operation %q as owned by another operation", observation.OperationID) + } + default: + return fmt.Errorf("adapter returned unsupported rotation guard status %q", observation.Status) + } + return nil +} + +func (o Orchestrator) acquireRotationGuard(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + reference, err := checkpointRotationGuard(*checkpoint) + if err != nil { + return err + } + observation, err := o.observeRotationGuard(ctx, reference) + if err != nil { + return fmt.Errorf("observe signer-rotation guard before acquisition: %w", err) + } + switch observation.Status { + case RotationGuardHeld: + return o.persistAcquiredRotationGuard(ctx, workspace, checkpoint, reference) + case RotationGuardCompleted: + return rotationGuardConflict(checkpoint.Phase, reference, "the exact signer-rotation operation was already completed before this checkpoint acquired it") + case RotationGuardOwnedByOther: + return rotationGuardOwnedByOther(checkpoint.Phase, reference, observation.OperationID) + case RotationGuardNotFound: + } + + outcome, acquireErr := o.Cluster.AcquireRotationGuard(ctx, reference) + if err := validateEffectOutcome(outcome); err != nil { + return err + } + observed, observeErr := o.observeRotationGuard(ctx, reference) + if observeErr == nil { + switch observed.Status { + case RotationGuardHeld: + return o.persistAcquiredRotationGuard(ctx, workspace, checkpoint, reference) + case RotationGuardOwnedByOther: + return rotationGuardOwnedByOther(checkpoint.Phase, reference, observed.OperationID) + case RotationGuardCompleted: + return rotationGuardConflict(checkpoint.Phase, reference, "the exact signer-rotation operation completed while acquisition was being reconciled") + case RotationGuardNotFound: + } + } + + switch outcome { + case EffectUnknown, EffectSubmitted: + return &OutcomeUnknownError{ + Phase: checkpoint.Phase, + Operation: "acquire cluster-wide signer-rotation guard", + Cause: errors.Join(acquireErr, observeErr), + } + case EffectNotApplied: + if acquireErr != nil { + return fmt.Errorf("signer-rotation guard acquisition was not applied: %w", acquireErr) + } + if observeErr != nil { + return fmt.Errorf("reconcile unapplied signer-rotation guard acquisition: %w", observeErr) + } + return rotationGuardConflict(checkpoint.Phase, reference, "the signer-rotation guard acquisition was not applied") + default: + return fmt.Errorf("unsupported signer-rotation guard acquisition outcome %q", outcome) + } +} + +func (o Orchestrator) persistAcquiredRotationGuard(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint, reference RotationGuardReference) error { + state, err := o.observeStableSignerState(ctx, checkpoint.Phase, &reference) + if err != nil { + return fmt.Errorf("observe signer state after acquiring rotation guard: %w", err) + } + if err := validateRecordedSignerState(checkpoint.Phase, state, *checkpoint); err != nil { + return err + } + return advanceCheckpoint(workspace, checkpoint, PhaseGuardAcquired, nil) +} + +func (o Orchestrator) requireRotationGuardHeld(ctx context.Context, checkpoint Checkpoint) error { + reference, err := checkpointRotationGuard(checkpoint) + if err != nil { + return err + } + observation, err := o.observeRotationGuard(ctx, reference) + if err != nil { + return fmt.Errorf("observe signer-rotation guard ownership: %w", err) + } + switch observation.Status { + case RotationGuardHeld: + return nil + case RotationGuardOwnedByOther: + return rotationGuardOwnedByOther(checkpoint.Phase, reference, observation.OperationID) + case RotationGuardCompleted: + return rotationGuardConflict(checkpoint.Phase, reference, "the signer-rotation guard was released before this checkpoint reached its release phase") + case RotationGuardNotFound: + return rotationGuardConflict(checkpoint.Phase, reference, "the signer-rotation guard is no longer observable") + default: + return fmt.Errorf("unsupported signer-rotation guard status %q", observation.Status) + } +} + +func (o Orchestrator) releaseRotationGuard(ctx context.Context, workspace *RotationWorkspace, checkpoint *Checkpoint) error { + reference, err := checkpointRotationGuard(*checkpoint) + if err != nil { + return err + } + observation, err := o.observeRotationGuard(ctx, reference) + if err != nil { + return fmt.Errorf("observe signer-rotation guard before release: %w", err) + } + switch observation.Status { + case RotationGuardCompleted: + return advanceCheckpoint(workspace, checkpoint, PhaseComplete, nil) + case RotationGuardOwnedByOther: + return rotationGuardOwnedByOther(checkpoint.Phase, reference, observation.OperationID) + case RotationGuardNotFound: + return rotationGuardConflict(checkpoint.Phase, reference, "the signer-rotation guard disappeared before a durable completion record was observed") + case RotationGuardHeld: + } + + outcome, releaseErr := o.Cluster.ReleaseRotationGuard(ctx, reference) + if err := validateEffectOutcome(outcome); err != nil { + return err + } + observed, observeErr := o.observeRotationGuard(ctx, reference) + if observeErr == nil { + switch observed.Status { + case RotationGuardCompleted: + return advanceCheckpoint(workspace, checkpoint, PhaseComplete, nil) + case RotationGuardOwnedByOther: + return rotationGuardOwnedByOther(checkpoint.Phase, reference, observed.OperationID) + case RotationGuardNotFound, RotationGuardHeld: + } + } + + switch outcome { + case EffectUnknown, EffectSubmitted: + return &OutcomeUnknownError{ + Phase: checkpoint.Phase, + Operation: "release cluster-wide signer-rotation guard", + Cause: errors.Join(releaseErr, observeErr), + } + case EffectNotApplied: + if releaseErr != nil { + return fmt.Errorf("signer-rotation guard release was not applied: %w", releaseErr) + } + if observeErr != nil { + return fmt.Errorf("reconcile unapplied signer-rotation guard release: %w", observeErr) + } + return rotationGuardConflict(checkpoint.Phase, reference, "the signer-rotation guard release was not applied") + default: + return fmt.Errorf("unsupported signer-rotation guard release outcome %q", outcome) + } +} + +func (o Orchestrator) reconcileReleasedRotationGuard(ctx context.Context, checkpoint Checkpoint) error { + reference, err := checkpointRotationGuard(checkpoint) + if err != nil { + return err + } + observation, err := o.observeRotationGuard(ctx, reference) + if err != nil { + return fmt.Errorf("reconcile signer-rotation guard completion: %w", err) + } + if observation.Status == RotationGuardOwnedByOther { + return rotationGuardOwnedByOther(checkpoint.Phase, reference, observation.OperationID) + } + if checkpoint.Phase == PhaseComplete && observation.Status != RotationGuardCompleted { + return rotationGuardConflict(checkpoint.Phase, reference, "the completed checkpoint has no durable signer-rotation guard completion record") + } + if checkpoint.Phase == PhaseGuardReleaseRecorded && observation.Status != RotationGuardHeld && observation.Status != RotationGuardCompleted { + return rotationGuardConflict(checkpoint.Phase, reference, "the signer-rotation guard is neither held nor durably completed at the release checkpoint") + } + return nil +} + +func checkpointRotationGuard(checkpoint Checkpoint) (RotationGuardReference, error) { + if checkpoint.RotationGuard == nil { + return RotationGuardReference{}, fmt.Errorf("checkpoint is missing the signer-rotation guard reference") + } + reference := *checkpoint.RotationGuard + if err := validateRotationGuardReference(reference); err != nil { + return RotationGuardReference{}, err + } + return reference, nil +} + +func rotationGuardOwnedByOther(phase Phase, reference RotationGuardReference, owner string) error { + return &ConflictError{ + Phase: phase, + Reason: fmt.Sprintf("signer-rotation guard scope %q is owned by different operation %q instead of %q", reference.ScopeID, owner, reference.OperationID), + } +} + +func rotationGuardConflict(phase Phase, reference RotationGuardReference, reason string) error { + return &ConflictError{ + Phase: phase, + Reason: fmt.Sprintf("signer-rotation guard operation %q: %s", reference.OperationID, reason), + } +} diff --git a/pkg/cmd/provisioning/rotation/rotation_guard_test.go b/pkg/cmd/provisioning/rotation/rotation_guard_test.go new file mode 100644 index 0000000000..d241aea148 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/rotation_guard_test.go @@ -0,0 +1,290 @@ +package rotation + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" +) + +func TestRotationGuardReferenceUsesClusterGlobalScopeAndExactOperationEvidence(t *testing.T) { + baseline := clonePublicSignerBaseline(*testSignerBaseline()) + signerRef := *testSignerObjectReference() + + first, err := deriveRotationGuardReference("cluster-123", ProviderAWS, "aws://issuer-a", baseline, signerRef) + if err != nil { + t.Fatalf("derive first guard reference: %v", err) + } + repeated, err := deriveRotationGuardReference("cluster-123", ProviderAWS, "aws://issuer-a", clonePublicSignerBaseline(baseline), signerRef) + if err != nil { + t.Fatalf("derive repeated guard reference: %v", err) + } + if first != repeated { + t.Fatalf("deterministic guard references differ: first=%#v repeated=%#v", first, repeated) + } + + wrongTarget, err := deriveRotationGuardReference("cluster-123", ProviderGCP, "gcp://issuer-b", baseline, signerRef) + if err != nil { + t.Fatalf("derive different target guard reference: %v", err) + } + if wrongTarget.ScopeID != first.ScopeID { + t.Fatalf("one cluster's signer rotations received different scopes: %q and %q", first.ScopeID, wrongTarget.ScopeID) + } + if wrongTarget.OperationID == first.OperationID { + t.Fatal("different provider and target identities received the same operation ID") + } + + changedBaseline := clonePublicSignerBaseline(baseline) + changedBaseline.ConfigMapResourceVersion = "67891" + fromChangedBaseline, err := deriveRotationGuardReference("cluster-123", ProviderAWS, "aws://issuer-a", changedBaseline, signerRef) + if err != nil { + t.Fatalf("derive changed-baseline guard reference: %v", err) + } + if fromChangedBaseline.ScopeID != first.ScopeID || fromChangedBaseline.OperationID == first.OperationID { + t.Fatalf("baseline change produced unexpected guard reference: first=%#v changed=%#v", first, fromChangedBaseline) + } + + changedSignerRef := signerRef + changedSignerRef.ResourceVersion = "12346" + fromChangedSigner, err := deriveRotationGuardReference("cluster-123", ProviderAWS, "aws://issuer-a", baseline, changedSignerRef) + if err != nil { + t.Fatalf("derive changed-signer guard reference: %v", err) + } + if fromChangedSigner.ScopeID != first.ScopeID || fromChangedSigner.OperationID == first.OperationID { + t.Fatalf("signer reference change produced unexpected guard reference: first=%#v changed=%#v", first, fromChangedSigner) + } + + otherCluster, err := deriveRotationGuardReference("cluster-456", ProviderAWS, "aws://issuer-a", baseline, signerRef) + if err != nil { + t.Fatalf("derive other-cluster guard reference: %v", err) + } + if otherCluster.ScopeID == first.ScopeID || otherCluster.OperationID == first.OperationID { + t.Fatalf("different clusters shared guard identity: first=%#v other=%#v", first, otherCluster) + } +} + +func TestOrchestratorReconcilesUnknownGuardAcquisitionWithoutRepeatingMutation(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeManual) + cluster := &faultingGuardCluster{ + fakeClusterRotation: harness.cluster, + unknownAcquire: true, + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeManual, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("initial Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhasePreflightComplete || result.Complete { + t.Fatalf("initial Run() result = %#v, want preflight checkpoint", result) + } + if harness.cluster.guardAcquireCalls != 1 || harness.cluster.guardAcquireMutations != 1 { + t.Fatalf("guard acquisition after unknown outcome = calls %d, mutations %d; want 1, 1", harness.cluster.guardAcquireCalls, harness.cluster.guardAcquireMutations) + } + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("load interrupted checkpoint: %v", loadErr) + } + if checkpoint.RotationGuard == nil { + t.Fatal("guard reference was not durable before acquisition mutation") + } + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + var pause *PauseError + if !errors.As(err, &pause) || pause.Reason != PauseForCurrentJWKS { + t.Fatalf("resumed Run() error = %v, want current-JWKS pause", err) + } + if result.Phase != PhaseGuardAcquired || result.Complete { + t.Fatalf("resumed Run() result = %#v, want guard-acquired checkpoint", result) + } + if harness.cluster.guardAcquireCalls != 1 || harness.cluster.guardAcquireMutations != 1 { + t.Fatalf("resume repeated guard mutation: calls %d, mutations %d", harness.cluster.guardAcquireCalls, harness.cluster.guardAcquireMutations) + } +} + +func TestOrchestratorFailsBeforeSignerObservationWhenGuardIsLostOrOwnedByOther(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*fakeClusterRotation, RotationGuardReference) + }{ + { + name: "guard disappeared", + mutate: func(cluster *fakeClusterRotation, _ RotationGuardReference) { + cluster.guardReference = nil + }, + }, + { + name: "guard owned by another operation", + mutate: func(cluster *fakeClusterRotation, reference RotationGuardReference) { + reference.OperationID = strings.Repeat("b", 64) + cluster.guardReference = &reference + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeManual) + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeManual, OutputDir: outputDir} + result, err := harness.orchestrator.Run(context.Background(), options) + var pause *PauseError + if !errors.As(err, &pause) || result.Phase != PhaseGuardAcquired { + t.Fatalf("initial Run() result = %#v, error = %v; want guarded current-JWKS pause", result, err) + } + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("load guarded checkpoint: %v", loadErr) + } + test.mutate(harness.cluster, *checkpoint.RotationGuard) + harness.events = nil + + options.Resume = true + options.Manual.CurrentJWKS = append([]byte(nil), harness.currentJWKS...) + result, err = harness.orchestrator.Run(context.Background(), options) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("resumed Run() error = %v, want ConflictError", err) + } + if result.Phase != PhaseGuardAcquired || result.Complete { + t.Fatalf("resumed Run() result = %#v, want no advancement", result) + } + if !reflect.DeepEqual(harness.events, []string{"cluster.observe-rotation-guard"}) { + t.Fatalf("events after guard loss = %v, want only exact guard observation", harness.events) + } + }) + } +} + +func TestOrchestratorReconcilesReleasedGuardAfterLaterOperationAcquiresScope(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + cluster := &faultingGuardCluster{ + fakeClusterRotation: harness.cluster, + unknownRelease: true, + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("initial Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhaseGuardReleaseRecorded || result.Complete { + t.Fatalf("initial Run() result = %#v, want durable guard-release checkpoint", result) + } + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("load release checkpoint: %v", loadErr) + } + if _, completed := harness.cluster.completedGuards[checkpoint.RotationGuard.OperationID]; !completed { + t.Fatal("release mutation did not leave the exact durable completion record") + } + other := *checkpoint.RotationGuard + other.OperationID = strings.Repeat("c", 64) + harness.cluster.guardReference = &other + harness.events = nil + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if err != nil { + t.Fatalf("resumed Run() returned unexpected error: %v", err) + } + if result.Phase != PhaseComplete || !result.Complete { + t.Fatalf("resumed Run() result = %#v, want complete", result) + } + if !reflect.DeepEqual(harness.events, []string{"cluster.observe-rotation-guard", "cluster.observe-rotation-guard"}) { + t.Fatalf("terminal resume events = %v, want exact completion observations only", harness.events) + } +} + +func TestSecondClusterRotationWithDifferentTargetIsBlockedBySharedScope(t *testing.T) { + old := safetySigner(testPublicKeyPEM(t), "old-uid", "10") + replacement := safetySigner(testPublicKeyPEM(t), "new-uid", "20") + registry := newSharedSignerRegistry(old, replacement) + firstCluster := &sharedSignerCluster{safetyCluster: newSafetyCluster(old, replacement), registry: registry} + secondCluster := &sharedSignerCluster{safetyCluster: newSafetyCluster(old, replacement), registry: registry} + + firstDir := t.TempDir() + firstResult, firstErr := (Orchestrator{Cluster: firstCluster, Target: safetyTarget("issuer-a")}).Run(context.Background(), RunOptions{ + Provider: ProviderAWS, PublicationMode: PublicationModeManual, OutputDir: firstDir, + }) + var pause *PauseError + if !errors.As(firstErr, &pause) || firstResult.Phase != PhaseGuardAcquired { + t.Fatalf("first Run() result = %#v, error = %v; want guarded pause", firstResult, firstErr) + } + + secondDir := t.TempDir() + secondResult, secondErr := (Orchestrator{Cluster: secondCluster, Target: safetyTarget("issuer-b")}).Run(context.Background(), RunOptions{ + Provider: ProviderGCP, PublicationMode: PublicationModeManual, OutputDir: secondDir, + }) + var conflict *ConflictError + if !errors.As(secondErr, &conflict) { + t.Fatalf("second Run() error = %v, want ConflictError", secondErr) + } + if secondResult.Phase != PhasePreflightComplete || secondResult.Complete { + t.Fatalf("second Run() result = %#v, want blocked preflight checkpoint", secondResult) + } + + firstCheckpoint, err := LoadCheckpoint(firstDir) + if err != nil { + t.Fatalf("load first checkpoint: %v", err) + } + secondCheckpoint, err := LoadCheckpoint(secondDir) + if err != nil { + t.Fatalf("load second checkpoint: %v", err) + } + if firstCheckpoint.RotationGuard.ScopeID != secondCheckpoint.RotationGuard.ScopeID { + t.Fatalf("same cluster used different guard scopes: first=%#v second=%#v", firstCheckpoint.RotationGuard, secondCheckpoint.RotationGuard) + } + if firstCheckpoint.RotationGuard.OperationID == secondCheckpoint.RotationGuard.OperationID { + t.Fatal("different provider/target evidence produced the same operation ID") + } + acquireCalls, acquireMutations, _, _ := registry.guardCounts() + if acquireCalls != 1 || acquireMutations != 1 { + t.Fatalf("shared guard acquisition = calls %d, mutations %d; want first operation only", acquireCalls, acquireMutations) + } + if requests := registry.requestCount(); requests != 0 { + t.Fatalf("signer replacement requests = %d, want zero before the blocked operation", requests) + } +} + +type faultingGuardCluster struct { + *fakeClusterRotation + unknownAcquire bool + unknownRelease bool + failNextObserve error +} + +func (cluster *faultingGuardCluster) ObserveRotationGuard(ctx context.Context, reference RotationGuardReference) (RotationGuardObservation, error) { + if cluster.failNextObserve != nil { + err := cluster.failNextObserve + cluster.failNextObserve = nil + return RotationGuardObservation{}, err + } + return cluster.fakeClusterRotation.ObserveRotationGuard(ctx, reference) +} + +func (cluster *faultingGuardCluster) AcquireRotationGuard(ctx context.Context, reference RotationGuardReference) (EffectOutcome, error) { + outcome, err := cluster.fakeClusterRotation.AcquireRotationGuard(ctx, reference) + if cluster.unknownAcquire { + cluster.unknownAcquire = false + cluster.failNextObserve = errors.New("guard acquisition readback interrupted") + return EffectUnknown, errors.New("guard acquisition response lost") + } + return outcome, err +} + +func (cluster *faultingGuardCluster) ReleaseRotationGuard(ctx context.Context, reference RotationGuardReference) (EffectOutcome, error) { + outcome, err := cluster.fakeClusterRotation.ReleaseRotationGuard(ctx, reference) + if cluster.unknownRelease { + cluster.unknownRelease = false + cluster.failNextObserve = errors.New("guard release readback interrupted") + return EffectUnknown, errors.New("guard release response lost") + } + return outcome, err +} diff --git a/pkg/cmd/provisioning/rotation/signer_state.go b/pkg/cmd/provisioning/rotation/signer_state.go new file mode 100644 index 0000000000..12229f3519 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/signer_state.go @@ -0,0 +1,246 @@ +package rotation + +import ( + "context" + "fmt" + "reflect" + "sort" + + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" +) + +const stableSignerObservationAttempts = 3 + +type stableSignerState struct { + Reference *SignerObjectReference + Baseline PublicSignerBaseline + publicPEM map[string][]byte +} + +type signerStateProgress int + +const ( + signerStateOriginal signerStateProgress = iota + signerStateReplacementRequested + signerStateReplacementReady +) + +type replacementSignerCandidate struct { + Evidence ReplacementSignerEvidence + PublicPEM []byte +} + +// observeStableSignerState makes the metadata-only Secret reference reads +// surrounding a public ConfigMap read visible to and enforceable by the shared +// engine. A result is accepted only when the same Secret object state bounded +// the complete public bundle observation. +func (o Orchestrator) observeStableSignerState(ctx context.Context, phase Phase, guard *RotationGuardReference) (stableSignerState, error) { + for attempt := 0; attempt < stableSignerObservationAttempts; attempt++ { + before, err := o.Cluster.ObserveSignerReference(ctx, guard) + if err != nil { + return stableSignerState{}, fmt.Errorf("observe signer Secret metadata before public bundle: %w", err) + } + before, err = validatedSignerReference(before) + if err != nil { + return stableSignerState{}, fmt.Errorf("validate signer Secret metadata before public bundle: %w", err) + } + + bundle, err := o.Cluster.ObservePublicSignerBundle(ctx, guard) + if err != nil { + return stableSignerState{}, fmt.Errorf("observe public signer bundle: %w", err) + } + state, err := normalizePublicSignerBundle(bundle) + if err != nil { + return stableSignerState{}, fmt.Errorf("validate public signer bundle: %w", err) + } + + after, err := o.Cluster.ObserveSignerReference(ctx, guard) + if err != nil { + return stableSignerState{}, fmt.Errorf("observe signer Secret metadata after public bundle: %w", err) + } + after, err = validatedSignerReference(after) + if err != nil { + return stableSignerState{}, fmt.Errorf("validate signer Secret metadata after public bundle: %w", err) + } + if reflect.DeepEqual(before, after) { + state.Reference = after + return state, nil + } + } + return stableSignerState{}, &ConflictError{ + Phase: phase, + Reason: "the signer Secret changed while reading the public signer bundle", + } +} + +func validatedSignerReference(reference *SignerObjectReference) (*SignerObjectReference, error) { + if reference == nil { + return nil, nil + } + if err := validateSignerObjectReference(*reference); err != nil { + return nil, err + } + clone := *reference + return &clone, nil +} + +func normalizePublicSignerBundle(observation PublicSignerBundleObservation) (stableSignerState, error) { + baseline := PublicSignerBaseline{ + ConfigMapUID: observation.ConfigMapUID, + ConfigMapResourceVersion: observation.ConfigMapResourceVersion, + Entries: make([]PublicSignerBaselineEntry, 0, len(observation.Signers)), + } + if err := validateOpaqueCheckpointValue("public signer ConfigMap UID", baseline.ConfigMapUID); err != nil { + return stableSignerState{}, err + } + if err := validateOpaqueCheckpointValue("public signer ConfigMap resource version", baseline.ConfigMapResourceVersion); err != nil { + return stableSignerState{}, err + } + if len(observation.Signers) == 0 { + return stableSignerState{}, fmt.Errorf("public signer ConfigMap must contain at least one entry") + } + + publicPEM := make(map[string][]byte, len(observation.Signers)) + for _, signer := range observation.Signers { + if !publicSignerEntryNamePattern.MatchString(signer.Name) { + return stableSignerState{}, fmt.Errorf("public signer entry name %q is not supported", signer.Name) + } + if _, exists := publicPEM[signer.Name]; exists { + return stableSignerState{}, fmt.Errorf("public signer ConfigMap contains duplicate entry name %q", signer.Name) + } + keySet, err := jwkutil.NewSigner(signer.PublicKeyPEM) + if err != nil { + return stableSignerState{}, fmt.Errorf("parse public signer entry %q: %w", signer.Name, err) + } + publicPEM[signer.Name] = append([]byte(nil), signer.PublicKeyPEM...) + baseline.Entries = append(baseline.Entries, PublicSignerBaselineEntry{ + Name: signer.Name, + SHA256: publicDigest(signer.PublicKeyPEM), + KeyID: keySet.Keys[0].KeyID, + }) + } + sort.Slice(baseline.Entries, func(i, j int) bool { + return baseline.Entries[i].Name < baseline.Entries[j].Name + }) + if err := validatePublicSignerBaseline(baseline); err != nil { + return stableSignerState{}, err + } + return stableSignerState{Baseline: baseline, publicPEM: publicPEM}, nil +} + +func classifySignerState(phase Phase, state stableSignerState, checkpoint Checkpoint) (signerStateProgress, *replacementSignerCandidate, error) { + if checkpoint.PreRotationSignerBaseline == nil || checkpoint.PreRotationSignerRef == nil { + return signerStateOriginal, nil, fmt.Errorf("checkpoint is missing pre-rotation signer evidence") + } + baseline := checkpoint.PreRotationSignerBaseline + if state.Baseline.ConfigMapUID != baseline.ConfigMapUID { + return signerStateOriginal, nil, signerStateConflict(phase, "the public signer ConfigMap UID changed") + } + + observedByName := make(map[string]PublicSignerBaselineEntry, len(state.Baseline.Entries)) + for _, entry := range state.Baseline.Entries { + observedByName[entry.Name] = entry + } + baselineByName := make(map[string]PublicSignerBaselineEntry, len(baseline.Entries)) + for _, entry := range baseline.Entries { + baselineByName[entry.Name] = entry + observed, exists := observedByName[entry.Name] + if !exists { + return signerStateOriginal, nil, signerStateConflict(phase, fmt.Sprintf("pre-rotation public signer entry %q disappeared", entry.Name)) + } + if observed != entry { + return signerStateOriginal, nil, signerStateConflict(phase, fmt.Sprintf("pre-rotation public signer entry %q changed", entry.Name)) + } + } + + var appended *PublicSignerBaselineEntry + for _, entry := range state.Baseline.Entries { + if _, existed := baselineByName[entry.Name]; existed { + continue + } + if appended != nil { + return signerStateOriginal, nil, signerStateConflict(phase, "more than one public signer entry was appended after the pre-rotation snapshot") + } + entryCopy := entry + appended = &entryCopy + } + if appended == nil { + if state.Baseline.ConfigMapResourceVersion != baseline.ConfigMapResourceVersion { + return signerStateOriginal, nil, signerStateConflict(phase, "the public signer ConfigMap resource version changed without one appended signer") + } + } else if state.Baseline.ConfigMapResourceVersion == baseline.ConfigMapResourceVersion { + return signerStateOriginal, nil, signerStateConflict(phase, "the public signer ConfigMap changed without a new resource version") + } + + oldReference := *checkpoint.PreRotationSignerRef + switch { + case state.Reference == nil: + return signerStateReplacementRequested, nil, nil + case state.Reference.UID == oldReference.UID: + if *state.Reference != oldReference { + return signerStateOriginal, nil, signerStateConflict(phase, "the pre-rotation signer Secret resource version changed without replacement") + } + if appended != nil { + return signerStateOriginal, nil, signerStateConflict(phase, "a public signer was appended while the pre-rotation signer Secret remained current") + } + return signerStateOriginal, nil, nil + case appended == nil: + return signerStateReplacementRequested, nil, nil + default: + candidate := &replacementSignerCandidate{ + Evidence: ReplacementSignerEvidence{ + Entry: *appended, + SecretRef: *state.Reference, + }, + PublicPEM: append([]byte(nil), state.publicPEM[appended.Name]...), + } + return signerStateReplacementReady, candidate, nil + } +} + +func validateRecordedSignerState(phase Phase, state stableSignerState, checkpoint Checkpoint) error { + progress, candidate, err := classifySignerState(phase, state, checkpoint) + if err != nil { + return err + } + switch { + case phase == PhasePreflightComplete: + // A concurrent workspace may have requested the same replacement after + // this workspace persisted its baseline but before it supplied a manual + // current JWKS. The exact delta is still safe to adopt; artifact binding + // is enforced before this workspace can request or publish anything. + return nil + case phase == PhaseCurrentJWKSRead: + return nil + case phaseAtLeast(phase, PhaseNextPublicKeyRead): + if progress != signerStateReplacementReady || candidate == nil { + return signerStateConflict(phase, "the recorded replacement signer is not observable") + } + if checkpoint.ReplacementSigner == nil { + return fmt.Errorf("checkpoint is missing replacement signer evidence") + } + if candidate.Evidence != *checkpoint.ReplacementSigner { + return signerStateConflict(phase, "the observable replacement signer does not match the recorded evidence") + } + case phaseAtLeast(phase, PhaseNextKeyRequested): + if progress == signerStateOriginal { + return signerStateConflict(phase, "the pre-rotation signer Secret reappeared after replacement was recorded") + } + } + return nil +} + +func (o Orchestrator) revalidateRecordedSignerEvidence(ctx context.Context, checkpoint Checkpoint) error { + state, err := o.observeStableSignerState(ctx, checkpoint.Phase, checkpoint.RotationGuard) + if err != nil { + return fmt.Errorf("observe recorded replacement signer: %w", err) + } + if err := validateRecordedSignerState(checkpoint.Phase, state, checkpoint); err != nil { + return err + } + return nil +} + +func signerStateConflict(phase Phase, reason string) error { + return &ConflictError{Phase: phase, Reason: reason} +} diff --git a/pkg/cmd/provisioning/rotation/signer_state_test.go b/pkg/cmd/provisioning/rotation/signer_state_test.go new file mode 100644 index 0000000000..c60b0daea4 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/signer_state_test.go @@ -0,0 +1,545 @@ +package rotation + +import ( + "bytes" + "context" + "errors" + "reflect" + "strings" + "sync" + "testing" +) + +func TestNormalizePublicSignerBundleCanonicalizesAndRejectsAmbiguity(t *testing.T) { + oldPublic := testPublicKeyPEM(t) + newPublic := testPublicKeyPEM(t) + rsaLabel := bytes.ReplaceAll(oldPublic, []byte("PUBLIC KEY"), []byte("RSA PUBLIC KEY")) + + tests := []struct { + name string + signers []PublicSignerObservation + wantNames []string + wantError string + }{ + { + name: "canonicalizes entry order", + signers: []PublicSignerObservation{ + {Name: "service-account-002.pub", PublicKeyPEM: newPublic}, + {Name: "service-account-001.pub", PublicKeyPEM: oldPublic}, + }, + wantNames: []string{"service-account-001.pub", "service-account-002.pub"}, + }, + {name: "empty bundle", wantError: "at least one entry"}, + { + name: "unexpected entry name", + signers: []PublicSignerObservation{{Name: "active.pub", PublicKeyPEM: oldPublic}}, + wantError: "not supported", + }, + { + name: "duplicate entry name", + signers: []PublicSignerObservation{ + {Name: "service-account-001.pub", PublicKeyPEM: oldPublic}, + {Name: "service-account-001.pub", PublicKeyPEM: newPublic}, + }, + wantError: "duplicate entry name", + }, + { + name: "duplicate exact public value", + signers: []PublicSignerObservation{ + {Name: "service-account-001.pub", PublicKeyPEM: oldPublic}, + {Name: "service-account-002.pub", PublicKeyPEM: oldPublic}, + }, + wantError: "duplicate public value digest", + }, + { + name: "duplicate semantic key with different PEM bytes", + signers: []PublicSignerObservation{ + {Name: "service-account-001.pub", PublicKeyPEM: oldPublic}, + {Name: "service-account-002.pub", PublicKeyPEM: rsaLabel}, + }, + wantError: "duplicate key ID", + }, + { + name: "malformed public key", + signers: []PublicSignerObservation{{Name: "service-account-001.pub", PublicKeyPEM: []byte("not pem")}}, + wantError: "parse public signer entry", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + state, err := normalizePublicSignerBundle(PublicSignerBundleObservation{ + ConfigMapUID: "signer-configmap-uid", + ConfigMapResourceVersion: "100", + Signers: test.signers, + }) + if test.wantError != "" { + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("normalizePublicSignerBundle() error = %v, want %q", err, test.wantError) + } + return + } + if err != nil { + t.Fatalf("normalizePublicSignerBundle() returned unexpected error: %v", err) + } + gotNames := make([]string, len(state.Baseline.Entries)) + for index, entry := range state.Baseline.Entries { + gotNames[index] = entry.Name + } + if !reflect.DeepEqual(gotNames, test.wantNames) { + t.Fatalf("canonical entry names = %v, want %v", gotNames, test.wantNames) + } + }) + } +} + +func TestObserveStableSignerStateRequiresMatchingMetadataReads(t *testing.T) { + old := safetySigner(testPublicKeyPEM(t), "old-uid", "10") + replacement := safetySigner(testPublicKeyPEM(t), "new-uid", "11") + base := newSafetyCluster(old, replacement) + + t.Run("matching reference", func(t *testing.T) { + cluster := &scriptedSignerReferenceCluster{ + safetyCluster: base, + references: []*SignerObjectReference{ + cloneSignerReferenceForTest(&old.Reference), + cloneSignerReferenceForTest(&old.Reference), + }, + } + state, err := (Orchestrator{Cluster: cluster}).observeStableSignerState(context.Background(), PhaseInitialized, nil) + if err != nil { + t.Fatalf("observeStableSignerState() returned unexpected error: %v", err) + } + if state.Reference == nil || *state.Reference != old.Reference { + t.Fatalf("stable reference = %#v, want %#v", state.Reference, old.Reference) + } + }) + + t.Run("reference changes during every bundle read", func(t *testing.T) { + references := make([]*SignerObjectReference, 0, stableSignerObservationAttempts*2) + for attempt := 0; attempt < stableSignerObservationAttempts; attempt++ { + references = append(references, cloneSignerReferenceForTest(&old.Reference), cloneSignerReferenceForTest(&replacement.Reference)) + } + cluster := &scriptedSignerReferenceCluster{safetyCluster: base, references: references} + _, err := (Orchestrator{Cluster: cluster}).observeStableSignerState(context.Background(), PhaseCurrentJWKSRead, nil) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("observeStableSignerState() error = %v, want ConflictError", err) + } + }) +} + +func TestClassifySignerStateRequiresExactAppendAndChangedUID(t *testing.T) { + oldPublic := testPublicKeyPEM(t) + newPublic := testPublicKeyPEM(t) + thirdPublic := testPublicKeyPEM(t) + oldReference := SignerObjectReference{UID: "old-uid", ResourceVersion: "10"} + newReference := SignerObjectReference{UID: "new-uid", ResourceVersion: "20"} + oldSigner := PublicSignerObservation{Name: "service-account-001.pub", PublicKeyPEM: oldPublic} + newSigner := PublicSignerObservation{Name: "service-account-002.pub", PublicKeyPEM: newPublic} + thirdSigner := PublicSignerObservation{Name: "service-account-003.pub", PublicKeyPEM: thirdPublic} + initial := signerStateForTest(t, &oldReference, "configmap-uid", "100", oldSigner) + baseline := clonePublicSignerBaseline(initial.Baseline) + checkpoint := Checkpoint{PreRotationSignerBaseline: &baseline, PreRotationSignerRef: &oldReference} + + tests := []struct { + name string + state func(*testing.T) stableSignerState + wantProgress signerStateProgress + wantError string + }{ + {name: "original state", state: func(*testing.T) stableSignerState { return initial }, wantProgress: signerStateOriginal}, + {name: "Secret absent", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, nil, "configmap-uid", "100", oldSigner) + }, wantProgress: signerStateReplacementRequested}, + {name: "changed UID before append", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, &newReference, "configmap-uid", "100", oldSigner) + }, wantProgress: signerStateReplacementRequested}, + {name: "one append and changed UID", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, &newReference, "configmap-uid", "101", oldSigner, newSigner) + }, wantProgress: signerStateReplacementReady}, + {name: "same UID with changed resource version", state: func(t *testing.T) stableSignerState { + changed := SignerObjectReference{UID: oldReference.UID, ResourceVersion: "11"} + return signerStateForTest(t, &changed, "configmap-uid", "100", oldSigner) + }, wantError: "resource version changed"}, + {name: "append while old Secret remains", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, &oldReference, "configmap-uid", "101", oldSigner, newSigner) + }, wantError: "remained current"}, + {name: "old entry overwritten", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, &newReference, "configmap-uid", "101", PublicSignerObservation{Name: oldSigner.Name, PublicKeyPEM: newPublic}) + }, wantError: "changed"}, + {name: "old entry removed", state: func(t *testing.T) stableSignerState { + state := signerStateForTest(t, &newReference, "configmap-uid", "101", newSigner) + return state + }, wantError: "disappeared"}, + {name: "two entries appended", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, &newReference, "configmap-uid", "101", oldSigner, newSigner, thirdSigner) + }, wantError: "more than one"}, + {name: "ConfigMap recreated", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, &newReference, "other-configmap-uid", "101", oldSigner, newSigner) + }, wantError: "UID changed"}, + {name: "resource version changes without append", state: func(t *testing.T) stableSignerState { + return signerStateForTest(t, &newReference, "configmap-uid", "101", oldSigner) + }, wantError: "resource version changed without"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + progress, replacement, err := classifySignerState(PhaseCurrentJWKSRead, test.state(t), checkpoint) + if test.wantError != "" { + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("classifySignerState() error = %v, want %q", err, test.wantError) + } + return + } + if err != nil { + t.Fatalf("classifySignerState() returned unexpected error: %v", err) + } + if progress != test.wantProgress { + t.Fatalf("classifySignerState() progress = %v, want %v", progress, test.wantProgress) + } + if progress == signerStateReplacementReady { + if replacement == nil || replacement.Evidence.Entry.Name != newSigner.Name || replacement.Evidence.SecretRef.UID != newReference.UID || !bytes.Equal(replacement.PublicPEM, newPublic) { + t.Fatalf("replacement candidate = %#v", replacement) + } + } + }) + } +} + +func TestCheckpointRejectsReplacementWithPreRotationSecretUID(t *testing.T) { + checkpoint := completeTestCheckpoint() + checkpoint.ReplacementSigner.SecretRef.UID = checkpoint.PreRotationSignerRef.UID + if err := checkpoint.Validate(); err == nil || !strings.Contains(err.Error(), "must differ") { + t.Fatalf("Validate() error = %v, want changed-UID error", err) + } +} + +func TestValidateRecordedSignerStateRequiresExactReplacementReference(t *testing.T) { + old := safetySigner(testPublicKeyPEM(t), "old-uid", "10") + replacement := safetySigner(testPublicKeyPEM(t), "new-uid", "20") + initial := signerStateForTest(t, &old.Reference, "configmap-uid", "100", old.Signer) + baseline := clonePublicSignerBaseline(initial.Baseline) + checkpoint := Checkpoint{Phase: PhaseNextPublicKeyRead, PreRotationSignerBaseline: &baseline, PreRotationSignerRef: &old.Reference} + ready := signerStateForTest(t, &replacement.Reference, "configmap-uid", "101", old.Signer, replacement.Signer) + _, candidate, err := classifySignerState(checkpoint.Phase, ready, checkpoint) + if err != nil || candidate == nil { + t.Fatalf("classify ready signer state: candidate=%#v error=%v", candidate, err) + } + evidence := candidate.Evidence + checkpoint.ReplacementSigner = &evidence + + driftedReference := replacement.Reference + driftedReference.ResourceVersion = "21" + drifted := signerStateForTest(t, &driftedReference, "configmap-uid", "101", old.Signer, replacement.Signer) + err = validateRecordedSignerState(checkpoint.Phase, drifted, checkpoint) + var conflict *ConflictError + if !errors.As(err, &conflict) || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("validateRecordedSignerState() error = %v, want replacement-reference conflict", err) + } +} + +func TestOrchestratorRevalidatesReplacementAtEveryDisruptiveGate(t *testing.T) { + tests := []struct { + name string + driftAtObservation int + wantPhase Phase + wantPublicationCount int + }{ + {name: "before combined publication", driftAtObservation: 6, wantPhase: PhaseCombinedJWKSBuilt, wantPublicationCount: 0}, + {name: "after signer rollout", driftAtObservation: 7, wantPhase: PhaseCombinedJWKSPublished, wantPublicationCount: 1}, + {name: "before final publication", driftAtObservation: 8, wantPhase: PhasePostRebootStable, wantPublicationCount: 1}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + drift := PublicSignerObservation{Name: "service-account-003.pub", PublicKeyPEM: testPublicKeyPEM(t)} + cluster := &driftingSignerCluster{ + fakeClusterRotation: harness.cluster, + driftAtObservation: test.driftAtObservation, + driftReference: SignerObjectReference{ + UID: "second-replacement-uid", + ResourceVersion: "300", + }, + driftSigner: drift, + } + harness.orchestrator.Cluster = cluster + + result, err := harness.orchestrator.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: t.TempDir(), + }) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() error = %v, want ConflictError", err) + } + if result.Phase != test.wantPhase { + t.Fatalf("Run() phase = %q, want %q", result.Phase, test.wantPhase) + } + if len(harness.publisher.publications) != test.wantPublicationCount { + t.Fatalf("provider publication count = %d, want %d", len(harness.publisher.publications), test.wantPublicationCount) + } + }) + } +} + +func TestOrchestratorTwoPreflightedWorkspacesAdoptOneSignerReplacement(t *testing.T) { + old := safetySigner(testPublicKeyPEM(t), "old-uid", "10") + replacement := safetySigner(testPublicKeyPEM(t), "new-uid", "20") + registry := newSharedSignerRegistry(old, replacement) + currentJWKS := encodedJWKSForTest(t, old.Signer.PublicKeyPEM) + firstCluster := &sharedSignerCluster{safetyCluster: newSafetyCluster(old, replacement), registry: registry} + secondCluster := &sharedSignerCluster{safetyCluster: newSafetyCluster(old, replacement), registry: registry} + firstDir := t.TempDir() + secondDir := t.TempDir() + + for _, run := range []struct { + cluster *sharedSignerCluster + dir string + }{ + {cluster: firstCluster, dir: firstDir}, + {cluster: secondCluster, dir: secondDir}, + } { + result, err := (Orchestrator{Cluster: run.cluster, Target: safetyTarget("issuer")}).Run(context.Background(), RunOptions{ + Provider: ProviderAWS, PublicationMode: PublicationModeManual, OutputDir: run.dir, + }) + var pause *PauseError + if !errors.As(err, &pause) || result.Phase != PhaseGuardAcquired || pause.Reason != PauseForCurrentJWKS { + t.Fatalf("preflight Run() result = %#v, error = %v", result, err) + } + } + acquireCalls, acquireMutations, _, _ := registry.guardCounts() + if acquireCalls != 1 || acquireMutations != 1 { + t.Fatalf("shared guard acquisitions = calls %d, mutations %d; want one idempotent mutation", acquireCalls, acquireMutations) + } + firstPreflight, err := LoadCheckpoint(firstDir) + if err != nil { + t.Fatalf("load first guarded checkpoint: %v", err) + } + secondPreflight, err := LoadCheckpoint(secondDir) + if err != nil { + t.Fatalf("load second guarded checkpoint: %v", err) + } + if firstPreflight.RotationGuard == nil || !reflect.DeepEqual(firstPreflight.RotationGuard, secondPreflight.RotationGuard) { + t.Fatalf("identical workspaces did not converge on one guard: first=%#v second=%#v", firstPreflight.RotationGuard, secondPreflight.RotationGuard) + } + + for _, run := range []struct { + cluster *sharedSignerCluster + dir string + }{ + {cluster: firstCluster, dir: firstDir}, + {cluster: secondCluster, dir: secondDir}, + } { + result, err := (Orchestrator{Cluster: run.cluster, Target: safetyTarget("issuer")}).Run(context.Background(), RunOptions{ + Provider: ProviderAWS, PublicationMode: PublicationModeManual, OutputDir: run.dir, Resume: true, + Manual: ManualInput{CurrentJWKS: currentJWKS}, + }) + var pause *PauseError + if !errors.As(err, &pause) || result.Phase != PhaseCombinedJWKSBuilt || pause.Reason != PauseForPublication { + t.Fatalf("resumed Run() result = %#v, error = %v", result, err) + } + } + + if requests := registry.requestCount(); requests != 1 { + t.Fatalf("signer replacement requests = %d, want exactly one", requests) + } + first, err := LoadCheckpoint(firstDir) + if err != nil { + t.Fatalf("load first checkpoint: %v", err) + } + second, err := LoadCheckpoint(secondDir) + if err != nil { + t.Fatalf("load second checkpoint: %v", err) + } + if first.ReplacementSigner == nil || !reflect.DeepEqual(first.ReplacementSigner, second.ReplacementSigner) { + t.Fatalf("replacement evidence did not converge: first=%#v second=%#v", first.ReplacementSigner, second.ReplacementSigner) + } +} + +func signerStateForTest(t *testing.T, reference *SignerObjectReference, configMapUID, resourceVersion string, signers ...PublicSignerObservation) stableSignerState { + t.Helper() + state, err := normalizePublicSignerBundle(PublicSignerBundleObservation{ + ConfigMapUID: configMapUID, + ConfigMapResourceVersion: resourceVersion, + Signers: clonePublicSignerObservationsForTest(signers), + }) + if err != nil { + t.Fatalf("normalize signer state fixture: %v", err) + } + state.Reference = cloneSignerReferenceForTest(reference) + return state +} + +type scriptedSignerReferenceCluster struct { + *safetyCluster + references []*SignerObjectReference + next int +} + +func (cluster *scriptedSignerReferenceCluster) ObserveSignerReference(context.Context, *RotationGuardReference) (*SignerObjectReference, error) { + if cluster.next >= len(cluster.references) { + return nil, errors.New("reference script exhausted") + } + reference := cloneSignerReferenceForTest(cluster.references[cluster.next]) + cluster.next++ + return reference, nil +} + +type sharedSignerRegistry struct { + mutex sync.Mutex + reference *SignerObjectReference + bundle PublicSignerBundleObservation + replacement safetySignerFixture + requests int + guard *RotationGuardReference + completed map[string]struct{} + acquireCalls int + acquireMutations int + releaseCalls int + releaseMutations int +} + +func newSharedSignerRegistry(old, replacement safetySignerFixture) *sharedSignerRegistry { + return &sharedSignerRegistry{ + reference: cloneSignerReferenceForTest(&old.Reference), + bundle: PublicSignerBundleObservation{ + ConfigMapUID: "shared-signer-configmap-uid", + ConfigMapResourceVersion: "1000", + Signers: []PublicSignerObservation{clonePublicSignerObservationForTest(old.Signer)}, + }, + replacement: replacement, + } +} + +func (registry *sharedSignerRegistry) snapshot() (*SignerObjectReference, PublicSignerBundleObservation) { + registry.mutex.Lock() + defer registry.mutex.Unlock() + return cloneSignerReferenceForTest(registry.reference), clonePublicSignerBundleForTest(registry.bundle) +} + +func (registry *sharedSignerRegistry) request(reference SignerObjectReference) (EffectOutcome, error) { + registry.mutex.Lock() + defer registry.mutex.Unlock() + if registry.reference == nil || *registry.reference != reference { + return EffectNotApplied, nil + } + registry.requests++ + registry.reference = cloneSignerReferenceForTest(®istry.replacement.Reference) + registry.bundle.ConfigMapResourceVersion = "1001" + registry.bundle.Signers = append(registry.bundle.Signers, clonePublicSignerObservationForTest(registry.replacement.Signer)) + return EffectSubmitted, nil +} + +func (registry *sharedSignerRegistry) requestCount() int { + registry.mutex.Lock() + defer registry.mutex.Unlock() + return registry.requests +} + +func (registry *sharedSignerRegistry) observeGuard(reference RotationGuardReference) RotationGuardObservation { + registry.mutex.Lock() + defer registry.mutex.Unlock() + if _, completed := registry.completed[reference.OperationID]; completed { + return RotationGuardObservation{Status: RotationGuardCompleted, OperationID: reference.OperationID} + } + if registry.guard == nil { + return RotationGuardObservation{Status: RotationGuardNotFound} + } + if *registry.guard == reference { + return RotationGuardObservation{Status: RotationGuardHeld, OperationID: reference.OperationID} + } + return RotationGuardObservation{Status: RotationGuardOwnedByOther, OperationID: registry.guard.OperationID} +} + +func (registry *sharedSignerRegistry) acquireGuard(reference RotationGuardReference) (EffectOutcome, error) { + registry.mutex.Lock() + defer registry.mutex.Unlock() + registry.acquireCalls++ + if registry.guard == nil { + stored := reference + registry.guard = &stored + registry.acquireMutations++ + return EffectSubmitted, nil + } + if *registry.guard == reference { + return EffectSubmitted, nil + } + return EffectNotApplied, nil +} + +func (registry *sharedSignerRegistry) releaseGuard(reference RotationGuardReference) (EffectOutcome, error) { + registry.mutex.Lock() + defer registry.mutex.Unlock() + registry.releaseCalls++ + if _, completed := registry.completed[reference.OperationID]; completed { + return EffectSubmitted, nil + } + if registry.guard == nil || *registry.guard != reference { + return EffectNotApplied, nil + } + registry.guard = nil + if registry.completed == nil { + registry.completed = make(map[string]struct{}) + } + registry.completed[reference.OperationID] = struct{}{} + registry.releaseMutations++ + return EffectSubmitted, nil +} + +func (registry *sharedSignerRegistry) guardCounts() (int, int, int, int) { + registry.mutex.Lock() + defer registry.mutex.Unlock() + return registry.acquireCalls, registry.acquireMutations, registry.releaseCalls, registry.releaseMutations +} + +type sharedSignerCluster struct { + *safetyCluster + registry *sharedSignerRegistry +} + +func (cluster *sharedSignerCluster) ObserveRotationGuard(_ context.Context, reference RotationGuardReference) (RotationGuardObservation, error) { + return cluster.registry.observeGuard(reference), nil +} + +func (cluster *sharedSignerCluster) AcquireRotationGuard(_ context.Context, reference RotationGuardReference) (EffectOutcome, error) { + return cluster.registry.acquireGuard(reference) +} + +func (cluster *sharedSignerCluster) ReleaseRotationGuard(_ context.Context, reference RotationGuardReference) (EffectOutcome, error) { + return cluster.registry.releaseGuard(reference) +} + +type driftingSignerCluster struct { + *fakeClusterRotation + driftAtObservation int + driftReference SignerObjectReference + driftSigner PublicSignerObservation +} + +func (cluster *driftingSignerCluster) ObservePublicSignerBundle(ctx context.Context, guard *RotationGuardReference) (PublicSignerBundleObservation, error) { + if cluster.bundleObservations+1 == cluster.driftAtObservation { + cluster.nextReference = cloneSignerReferenceForTest(&cluster.driftReference) + cluster.nextBundle.ConfigMapResourceVersion = "1002" + cluster.nextBundle.Signers = append(cluster.nextBundle.Signers, clonePublicSignerObservationForTest(cluster.driftSigner)) + } + return cluster.fakeClusterRotation.ObservePublicSignerBundle(ctx, guard) +} + +func (cluster *sharedSignerCluster) ObserveSignerReference(context.Context, *RotationGuardReference) (*SignerObjectReference, error) { + reference, _ := cluster.registry.snapshot() + return reference, nil +} + +func (cluster *sharedSignerCluster) ObservePublicSignerBundle(context.Context, *RotationGuardReference) (PublicSignerBundleObservation, error) { + _, bundle := cluster.registry.snapshot() + return bundle, nil +} + +func (cluster *sharedSignerCluster) RequestReplacement(_ context.Context, _ RotationGuardReference, reference SignerObjectReference) (EffectOutcome, error) { + return cluster.registry.request(reference) +} + +func (cluster *sharedSignerCluster) WaitForReplacement(context.Context, RotationGuardReference, SignerObjectReference) error { + return nil +} diff --git a/pkg/cmd/provisioning/rotation/workspace.go b/pkg/cmd/provisioning/rotation/workspace.go new file mode 100644 index 0000000000..5984fc83e3 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/workspace.go @@ -0,0 +1,62 @@ +package rotation + +import ( + "fmt" + "path/filepath" + "strings" +) + +// RotationWorkspace represents an exclusively locked local rotation working +// directory. It is valid only for the duration of WithRotationWorkspace. +type RotationWorkspace struct { + outputDir string + active bool +} + +// WithRotationWorkspace holds one workspace lock for the complete operation. +// Orchestration code must include observation, external mutation, artifact +// persistence, and the following checkpoint update in the same callback. +func WithRotationWorkspace(outputDir string, operation func(*RotationWorkspace) error) error { + if operation == nil { + return fmt.Errorf("rotation workspace operation must not be nil") + } + resolvedOutputDir, err := resolveRotationOutputDir(outputDir) + if err != nil { + return err + } + if err := ensureCheckpointDirectory(resolvedOutputDir); err != nil { + return err + } + + return withCheckpointLock(resolvedOutputDir, func() error { + workspace := &RotationWorkspace{outputDir: resolvedOutputDir, active: true} + defer func() { workspace.active = false }() + return operation(workspace) + }) +} + +// OutputDir returns the canonical directory bound to this workspace lease. +func (w *RotationWorkspace) OutputDir() (string, error) { + if err := w.validateActive(); err != nil { + return "", err + } + return w.outputDir, nil +} + +func (w *RotationWorkspace) validateActive() error { + if w == nil || !w.active || w.outputDir == "" { + return fmt.Errorf("rotation workspace lease is not active") + } + return nil +} + +func resolveRotationOutputDir(outputDir string) (string, error) { + if strings.TrimSpace(outputDir) == "" { + return "", fmt.Errorf("rotation output directory must not be empty") + } + resolvedOutputDir, err := filepath.Abs(outputDir) + if err != nil { + return "", fmt.Errorf("resolve rotation output directory: %w", err) + } + return filepath.Clean(resolvedOutputDir), nil +} diff --git a/pkg/cmd/provisioning/rotation/workspace_test.go b/pkg/cmd/provisioning/rotation/workspace_test.go new file mode 100644 index 0000000000..97d26031c4 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/workspace_test.go @@ -0,0 +1,92 @@ +package rotation + +import ( + "errors" + "path/filepath" + "strings" + "testing" +) + +func TestWithRotationWorkspaceHoldsLockAcrossCheckpointOperations(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + var retained *RotationWorkspace + err := WithRotationWorkspace(outputDir, func(workspace *RotationWorkspace) error { + retained = workspace + resolved, err := workspace.OutputDir() + if err != nil { + return err + } + if !filepath.IsAbs(resolved) { + t.Fatalf("workspace output directory %q is not absolute", resolved) + } + + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, outputDir) + if err := workspace.SaveCheckpoint(checkpoint); err != nil { + t.Fatalf("workspace SaveCheckpoint() returned unexpected error: %v", err) + } + loaded, err := workspace.LoadCheckpoint() + if err != nil { + t.Fatalf("workspace LoadCheckpoint() returned unexpected error: %v", err) + } + if loaded.Phase != PhaseInitialized { + t.Fatalf("loaded phase = %q, want %q", loaded.Phase, PhaseInitialized) + } + currentRaw := encodedJWKSForTest(t, testPublicKeyPEM(t)) + currentMetadata, err := workspace.WriteArtifact(ArtifactCurrentJWKS, currentRaw) + if err != nil { + t.Fatalf("workspace WriteArtifact() returned unexpected error: %v", err) + } + loaded.Phase = PhasePreflightComplete + loaded.ClusterIdentity = "cluster-123" + loaded.TargetIdentity = "aws://issuer" + loaded.PreRotationSignerBaseline = testSignerBaselineForKeyID(currentMetadata.KeyIDs[0]) + loaded.PreRotationSignerRef = testSignerObjectReference() + setTestRotationGuard(&loaded) + if err := workspace.SaveCheckpoint(loaded); err != nil { + t.Fatalf("workspace SaveCheckpoint(preflight) returned unexpected error: %v", err) + } + loaded.Phase = PhaseGuardAcquired + if err := workspace.SaveCheckpoint(loaded); err != nil { + t.Fatalf("workspace SaveCheckpoint(guard acquired) returned unexpected error: %v", err) + } + loaded.Phase = PhaseCurrentJWKSRead + loaded.Artifacts = []ArtifactMetadata{currentMetadata} + if err := workspace.SaveCheckpoint(loaded); err != nil { + t.Fatalf("workspace SaveCheckpoint(current JWKS) returned unexpected error: %v", err) + } + + err = SaveCheckpoint(checkpoint) + if err == nil || !errors.Is(err, ErrCheckpointLocked) { + t.Fatalf("nested top-level SaveCheckpoint() error = %v, want ErrCheckpointLocked", err) + } + return nil + }) + if err != nil { + t.Fatalf("WithRotationWorkspace() returned unexpected error: %v", err) + } + + if _, err := retained.LoadCheckpoint(); err == nil || !strings.Contains(err.Error(), "lease is not active") { + t.Fatalf("retained workspace LoadCheckpoint() error = %v, want inactive-lease error", err) + } + if _, err := LoadCheckpoint(outputDir); err != nil { + t.Fatalf("top-level LoadCheckpoint() after lease returned unexpected error: %v", err) + } +} + +func TestRotationWorkspaceRejectsMismatchedCheckpointDirectory(t *testing.T) { + outputDir := filepath.Join(t.TempDir(), "rotation-output") + err := WithRotationWorkspace(outputDir, func(workspace *RotationWorkspace) error { + checkpoint := NewCheckpoint(ProviderAWS, PublicationModeDirect, filepath.Join(t.TempDir(), "other-output")) + return workspace.SaveCheckpoint(checkpoint) + }) + if err == nil || !strings.Contains(err.Error(), "does not match locked workspace") { + t.Fatalf("WithRotationWorkspace(mismatched checkpoint) error = %v", err) + } +} + +func TestWithRotationWorkspaceRejectsNilOperation(t *testing.T) { + err := WithRotationWorkspace(filepath.Join(t.TempDir(), "rotation-output"), nil) + if err == nil || !strings.Contains(err.Error(), "must not be nil") { + t.Fatalf("WithRotationWorkspace(nil) error = %v", err) + } +} diff --git a/pkg/cmd/provisioning/utils.go b/pkg/cmd/provisioning/utils.go index fe808ba9ab..165af27ddf 100644 --- a/pkg/cmd/provisioning/utils.go +++ b/pkg/cmd/provisioning/utils.go @@ -1,12 +1,7 @@ package provisioning import ( - "crypto" - "crypto/rsa" - "crypto/x509" - "encoding/base64" "encoding/json" - "encoding/pem" "fmt" "io" "log" @@ -22,6 +17,7 @@ import ( configv1 "github.com/openshift/api/config/v1" credreqv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" + jwkutil "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" ) type JSONWebKeySet struct { @@ -88,65 +84,25 @@ spec: func BuildJsonWebKeySet(publicKeyPath string) ([]byte, error) { log.Print("Reading public key") publicKeyContent, err := os.ReadFile(publicKeyPath) - if err != nil { return nil, errors.Wrap(err, "failed to read public key") } - block, _ := pem.Decode(publicKeyContent) - if block == nil { - return nil, errors.Wrap(err, "error decoding PEM file") - } - - publicKey, err := x509.ParsePKIXPublicKey(block.Bytes) - if err != nil { - return nil, errors.Wrap(err, "error parsing key content") - } - - var alg jose.SignatureAlgorithm - switch publicKey.(type) { - case *rsa.PublicKey: - alg = jose.RS256 - default: - return nil, errors.New("public key is not of type RSA") - } - - kid, err := KeyIDFromPublicKey(publicKey) + keySet, err := jwkutil.NewSigner(publicKeyContent) if err != nil { - return nil, errors.New("Failed to fetch key ID from public key") + return nil, errors.Wrap(err, "failed to build JSON web key set from public key") } - - var keys []jose.JSONWebKey - keys = append(keys, jose.JSONWebKey{ - Key: publicKey, - KeyID: kid, - Algorithm: string(alg), - Use: "sig", - }) - - keySet, err := json.MarshalIndent(JSONWebKeySet{Keys: keys}, "", " ") + encoded, err := json.MarshalIndent(JSONWebKeySet{Keys: keySet.Keys}, "", " ") if err != nil { - return nil, errors.New("JSON encoding of web key set failed") + return nil, errors.Wrap(err, "failed to encode JSON web key set") } - - return keySet, nil + return encoded, nil } // KeyIDFromPublicKey derives a key ID non-reversibly from a public key // reference: https://github.com/kubernetes/kubernetes/blob/0f140bf1eeaf63c155f5eba1db8db9b5d52d5467/pkg/serviceaccount/jwt.go#L89-L111 func KeyIDFromPublicKey(publicKey interface{}) (string, error) { - publicKeyDERBytes, err := x509.MarshalPKIXPublicKey(publicKey) - if err != nil { - return "", fmt.Errorf("failed to serialize public key to DER format: %v", err) - } - - hasher := crypto.SHA256.New() - hasher.Write(publicKeyDERBytes) - publicKeyDERHash := hasher.Sum(nil) - - keyID := base64.RawURLEncoding.EncodeToString(publicKeyDERHash) - - return keyID, nil + return jwkutil.KeyIDFromPublicKey(publicKey) } // GetListOfCredentialsRequests decodes manifests in a given directory and returns a list of CredentialsRequests diff --git a/pkg/cmd/provisioning/utils_test.go b/pkg/cmd/provisioning/utils_test.go index 1b4d5b74a1..a017781bb9 100644 --- a/pkg/cmd/provisioning/utils_test.go +++ b/pkg/cmd/provisioning/utils_test.go @@ -2,6 +2,7 @@ package provisioning import ( "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -94,6 +95,15 @@ func TestEnsureDir(t *testing.T) { } } +func TestBuildJsonWebKeySetRejectsInvalidPEM(t *testing.T) { + publicKeyPath := filepath.Join(t.TempDir(), "invalid-public-key.pem") + require.NoError(t, os.WriteFile(publicKeyPath, []byte("not PEM"), 0o600)) + + keySet, err := BuildJsonWebKeySet(publicKeyPath) + require.Error(t, err) + assert.Nil(t, keySet) +} + func TestFilteringCredReqs(t *testing.T) { tests := []struct { name string From 8eff1066e0816311c87ca6288e6ccf02a4dc549d Mon Sep 17 00:00:00 2001 From: Bruno Andrade Date: Mon, 7 Sep 2026 21:46:48 -0300 Subject: [PATCH 2/6] ccoctl: Address rotation review cleanups Keep pre-existing artifact relationship validation while removing a redundant equality check, and construct the checkpoint lock path with filepath.Join. Assisted-by: GPT-5 Signed-off-by: Bruno Andrade --- pkg/cmd/provisioning/rotation/artifact_store.go | 2 +- pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/cmd/provisioning/rotation/artifact_store.go b/pkg/cmd/provisioning/rotation/artifact_store.go index 4fe3085d48..bd8afde3d7 100644 --- a/pkg/cmd/provisioning/rotation/artifact_store.go +++ b/pkg/cmd/provisioning/rotation/artifact_store.go @@ -79,7 +79,7 @@ func (w *RotationWorkspace) WriteArtifact(name string, payload []byte) (Artifact if err := validateArtifactRelationships(checkpoint, payloads); err != nil { return ArtifactMetadata{}, err } - if existingErr == nil && bytes.Equal(existing, payload) { + if existingErr == nil { return metadata, nil } diff --git a/pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go b/pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go index 1db9bb9ad5..75a37eb7fb 100644 --- a/pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go +++ b/pkg/cmd/provisioning/rotation/checkpoint_lock_unix.go @@ -6,12 +6,13 @@ import ( "errors" "fmt" "os" + "path/filepath" "golang.org/x/sys/unix" ) func acquireCheckpointLock(outputDir string) (*os.File, error) { - lockPath := outputDir + string(os.PathSeparator) + checkpointLockFileName + lockPath := filepath.Join(outputDir, checkpointLockFileName) flags := unix.O_RDWR | unix.O_CLOEXEC | unix.O_NOFOLLOW fd, err := unix.Open(lockPath, flags|unix.O_CREAT|unix.O_EXCL, uint32(checkpointFileMode)) created := err == nil From 84aab9e858a0c45f0dcf7e08d80c585a8c3c857d Mon Sep 17 00:00:00 2001 From: Bruno Andrade Date: Tue, 8 Sep 2026 13:02:32 -0300 Subject: [PATCH 3/6] ccoctl: Expand rotation failure-path coverage Add focused tests for conditional publication, rotation guard, reboot, and cancellation outcomes without changing production behavior. Assisted-by: GPT-5 Signed-off-by: Bruno Andrade --- .../rotation/publication_outcomes_test.go | 162 ++++++++ .../rotation/reboot_cancellation_test.go | 288 +++++++++++++ .../rotation/rotation_guard_outcomes_test.go | 392 ++++++++++++++++++ 3 files changed, 842 insertions(+) create mode 100644 pkg/cmd/provisioning/rotation/publication_outcomes_test.go create mode 100644 pkg/cmd/provisioning/rotation/reboot_cancellation_test.go create mode 100644 pkg/cmd/provisioning/rotation/rotation_guard_outcomes_test.go diff --git a/pkg/cmd/provisioning/rotation/publication_outcomes_test.go b/pkg/cmd/provisioning/rotation/publication_outcomes_test.go new file mode 100644 index 0000000000..077a0de2ff --- /dev/null +++ b/pkg/cmd/provisioning/rotation/publication_outcomes_test.go @@ -0,0 +1,162 @@ +package rotation + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" +) + +func TestPublishDirectRejectsBlankRevisionWithoutPublishing(t *testing.T) { + predecessor := StoredArtifact{Data: []byte("predecessor")} + desired := StoredArtifact{Data: []byte("desired")} + publisher := &safetyPublisher{ + state: VersionedJWKS{Data: append([]byte(nil), predecessor.Data...), Revision: "7"}, + reads: []VersionedJWKS{{Data: predecessor.Data, Revision: " \t"}}, + } + orchestrator := Orchestrator{Publisher: publisher} + + err := orchestrator.publishDirect(context.Background(), Checkpoint{ + Phase: PhaseCombinedJWKSBuilt, + TargetIdentity: "target", + }, desired, predecessor) + if err == nil || !strings.Contains(err.Error(), "did not return the revision required for conditional publication") { + t.Fatalf("publishDirect() error = %v, want missing-revision error", err) + } + if publisher.publishCalls != 0 { + t.Fatalf("conditional publication calls = %d, want zero", publisher.publishCalls) + } + if !bytes.Equal(publisher.state.Data, predecessor.Data) { + t.Fatalf("provider state = %q, want unchanged predecessor %q", publisher.state.Data, predecessor.Data) + } +} + +func TestPublishDirectClassifiesRejectedConditionalPublication(t *testing.T) { + predecessor := StoredArtifact{Data: []byte("predecessor")} + desired := StoredArtifact{Data: []byte("desired")} + checkpoint := Checkpoint{Phase: PhaseCombinedJWKSBuilt, TargetIdentity: "target"} + publishErr := errors.New("conditional revision no longer matches") + + t.Run("publish error is preserved", func(t *testing.T) { + publisher := &safetyPublisher{ + state: VersionedJWKS{Data: append([]byte(nil), predecessor.Data...), Revision: "7"}, + publishOutcome: EffectNotApplied, + publishErr: publishErr, + } + orchestrator := Orchestrator{Publisher: publisher} + + err := orchestrator.publishDirect(context.Background(), checkpoint, desired, predecessor) + if !errors.Is(err, publishErr) || !strings.Contains(err.Error(), "was not applied") { + t.Fatalf("publishDirect() error = %v, want wrapped rejection error", err) + } + assertConditionalPublicationDidNotOverwrite(t, publisher, predecessor, 1) + }) + + t.Run("readback error is reported", func(t *testing.T) { + publisher := &safetyPublisher{ + state: VersionedJWKS{Data: append([]byte(nil), predecessor.Data...), Revision: "7"}, + failReadNumber: 2, + publishOutcome: EffectNotApplied, + } + orchestrator := Orchestrator{Publisher: publisher} + + err := orchestrator.publishDirect(context.Background(), checkpoint, desired, predecessor) + if err == nil || !strings.Contains(err.Error(), "read provider JWKS after rejected conditional publication") { + t.Fatalf("publishDirect() error = %v, want rejected-publication readback error", err) + } + assertConditionalPublicationDidNotOverwrite(t, publisher, predecessor, 1) + }) + + t.Run("unexpected readback is a conflict", func(t *testing.T) { + publisher := &safetyPublisher{ + state: VersionedJWKS{Data: append([]byte(nil), predecessor.Data...), Revision: "7"}, + reads: []VersionedJWKS{ + {Data: predecessor.Data, Revision: "7"}, + {Data: []byte("changed-by-another-writer"), Revision: "8"}, + }, + publishOutcome: EffectNotApplied, + } + orchestrator := Orchestrator{Publisher: publisher} + + err := orchestrator.publishDirect(context.Background(), checkpoint, desired, predecessor) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("publishDirect() error = %v, want ConflictError", err) + } + if conflict.Phase != checkpoint.Phase { + t.Fatalf("ConflictError phase = %q, want %q", conflict.Phase, checkpoint.Phase) + } + assertConditionalPublicationDidNotOverwrite(t, publisher, predecessor, 1) + }) +} + +func TestPublishDirectTreatsUnconfirmedSubmissionAsUnknown(t *testing.T) { + predecessor := StoredArtifact{Data: []byte("predecessor")} + desired := StoredArtifact{Data: []byte("desired")} + checkpoint := Checkpoint{Phase: PhaseCombinedJWKSBuilt, TargetIdentity: "target"} + + for _, outcome := range []EffectOutcome{EffectUnknown, EffectSubmitted} { + t.Run(string(outcome), func(t *testing.T) { + publisher := &safetyPublisher{ + state: VersionedJWKS{Data: append([]byte(nil), predecessor.Data...), Revision: "7"}, + reads: []VersionedJWKS{ + {Data: predecessor.Data, Revision: "7"}, + {Data: []byte("unexpected-readback"), Revision: "8"}, + }, + publishOutcome: outcome, + } + orchestrator := Orchestrator{Publisher: publisher} + + err := orchestrator.publishDirect(context.Background(), checkpoint, desired, predecessor) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("publishDirect() error = %v, want OutcomeUnknownError", err) + } + if unknown.Phase != checkpoint.Phase || unknown.Operation != "conditionally publish provider JWKS" { + t.Fatalf("OutcomeUnknownError = %#v, want phase %q and conditional publication operation", unknown, checkpoint.Phase) + } + assertConditionalPublicationDidNotOverwrite(t, publisher, predecessor, 1) + }) + } +} + +func TestPublishDirectDoesNotRepeatConfirmedSubmission(t *testing.T) { + predecessor := StoredArtifact{Data: []byte("predecessor")} + desired := StoredArtifact{Data: []byte("desired")} + checkpoint := Checkpoint{Phase: PhaseCombinedJWKSBuilt, TargetIdentity: "target"} + + for _, outcome := range []EffectOutcome{EffectUnknown, EffectSubmitted} { + t.Run(string(outcome), func(t *testing.T) { + publisher := &safetyPublisher{ + state: VersionedJWKS{Data: append([]byte(nil), predecessor.Data...), Revision: "7"}, + publishOutcome: outcome, + applyPublication: true, + } + orchestrator := Orchestrator{Publisher: publisher} + + if err := orchestrator.publishDirect(context.Background(), checkpoint, desired, predecessor); err != nil { + t.Fatalf("first publishDirect() error = %v", err) + } + if err := orchestrator.publishDirect(context.Background(), checkpoint, desired, predecessor); err != nil { + t.Fatalf("second publishDirect() error = %v", err) + } + if publisher.publishCalls != 1 { + t.Fatalf("conditional publication calls = %d, want one after confirmed retry", publisher.publishCalls) + } + if !bytes.Equal(publisher.state.Data, desired.Data) { + t.Fatalf("provider state = %q, want desired data %q", publisher.state.Data, desired.Data) + } + }) + } +} + +func assertConditionalPublicationDidNotOverwrite(t *testing.T, publisher *safetyPublisher, predecessor StoredArtifact, wantCalls int) { + t.Helper() + if publisher.publishCalls != wantCalls { + t.Fatalf("conditional publication calls = %d, want %d", publisher.publishCalls, wantCalls) + } + if !bytes.Equal(publisher.state.Data, predecessor.Data) { + t.Fatalf("provider state = %q, want unchanged predecessor %q", publisher.state.Data, predecessor.Data) + } +} diff --git a/pkg/cmd/provisioning/rotation/reboot_cancellation_test.go b/pkg/cmd/provisioning/rotation/reboot_cancellation_test.go new file mode 100644 index 0000000000..72f0278a60 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/reboot_cancellation_test.go @@ -0,0 +1,288 @@ +package rotation + +import ( + "context" + "errors" + "strings" + "testing" +) + +func TestOrchestratorCancellationBetweenPhasesIsDurableAndStopsBeforeNextEffect(t *testing.T) { + tests := []struct { + name string + cancelAfter string + wantPhase Phase + triggerEvent string + wantPublications []string + wantRebootRequest int + }{ + { + name: "after reboot intent is prepared", + cancelAfter: "prepare-reboot", + wantPhase: PhaseRebootIntentRecorded, + triggerEvent: "cluster.prepare-reboot", + wantPublications: []string{"combined"}, + wantRebootRequest: 0, + }, + { + name: "after post-reboot stability", + cancelAfter: "post-reboot-stable", + wantPhase: PhasePostRebootStable, + triggerEvent: "cluster.wait-for-post-reboot-stable", + wantPublications: []string{"combined"}, + wantRebootRequest: 1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cluster := &rebootCoverageCluster{ + fakeClusterRotation: harness.cluster, + cancel: cancel, + cancelAfter: test.cancelAfter, + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + + result, err := harness.orchestrator.Run(ctx, RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: outputDir, + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run() error = %v, want context.Canceled", err) + } + if result.Phase != test.wantPhase || result.Complete { + t.Fatalf("Run() result = %#v, want phase %q and incomplete", result, test.wantPhase) + } + + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("load cancelled checkpoint: %v", loadErr) + } + if checkpoint.Phase != test.wantPhase || checkpoint.LastErrorCode != errorCodeCancelled { + t.Fatalf("cancelled checkpoint phase/error = %q/%q, want %q/%q", checkpoint.Phase, checkpoint.LastErrorCode, test.wantPhase, errorCodeCancelled) + } + if !equalStrings(harness.publisher.publications, test.wantPublications) { + t.Fatalf("publications = %v, want %v", harness.publisher.publications, test.wantPublications) + } + if harness.cluster.rebootRequests != test.wantRebootRequest { + t.Fatalf("reboot requests = %d, want %d", harness.cluster.rebootRequests, test.wantRebootRequest) + } + trigger := eventIndex(harness.events, test.triggerEvent) + if trigger < 0 { + t.Fatalf("trigger event %q was not observed: %v", test.triggerEvent, harness.events) + } + if trigger != len(harness.events)-1 { + t.Fatalf("external events occurred after cancellation trigger %q: %v", test.triggerEvent, harness.events[trigger+1:]) + } + }) + } +} + +func TestOrchestratorRebootEffectNotAppliedDoesNotAdvanceOrWait(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + cluster := &rebootCoverageCluster{ + fakeClusterRotation: harness.cluster, + overrideRequest: true, + requestOutcome: EffectNotApplied, + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + + result, err := harness.orchestrator.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeDirect, + OutputDir: outputDir, + }) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() error = %v, want ConflictError", err) + } + if result.Phase != PhaseRebootIntentRecorded || result.Complete { + t.Fatalf("Run() result = %#v, want reboot-intent-recorded and incomplete", result) + } + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("load failed checkpoint: %v", loadErr) + } + if checkpoint.Phase != PhaseRebootIntentRecorded || checkpoint.LastErrorCode != errorCodeConflict { + t.Fatalf("checkpoint phase/error = %q/%q, want %q/%q", checkpoint.Phase, checkpoint.LastErrorCode, PhaseRebootIntentRecorded, errorCodeConflict) + } + if harness.cluster.rebootRequests != 1 || harness.cluster.canonicalReboot != nil { + t.Fatalf("reboot request count/canonical record = %d/%#v, want one rejected request and no mutation", harness.cluster.rebootRequests, harness.cluster.canonicalReboot) + } + if countExactEvent(harness.events, "cluster.wait-for-reboot") != 0 { + t.Fatalf("reboot wait ran after a confirmed unapplied request: %v", harness.events) + } +} + +func TestOrchestratorRebootReadbackFailureResumesWithoutDuplicateRequest(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + cluster := &rebootCoverageCluster{ + fakeClusterRotation: harness.cluster, + failReadbackAfterRequest: true, + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("initial Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhaseRebootIntentRecorded || harness.cluster.rebootRequests != 1 { + t.Fatalf("initial result/request count = %#v/%d, want reboot intent and one request", result, harness.cluster.rebootRequests) + } + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("load uncertain checkpoint: %v", loadErr) + } + if checkpoint.LastErrorCode != errorCodeExternalOutcomeUnknown { + t.Fatalf("last error code = %q, want %q", checkpoint.LastErrorCode, errorCodeExternalOutcomeUnknown) + } + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if err != nil { + t.Fatalf("resumed Run() returned unexpected error: %v", err) + } + if !result.Complete || result.Phase != PhaseComplete { + t.Fatalf("resumed Run() result = %#v, want complete", result) + } + if harness.cluster.rebootRequests != 1 { + t.Fatalf("reboot requests after resume = %d, want exactly one", harness.cluster.rebootRequests) + } +} + +func TestOrchestratorRejectsWaitThatReturnsBeforeCanonicalRebootCompletion(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + cluster := &rebootCoverageCluster{ + fakeClusterRotation: harness.cluster, + waitWithoutCompletion: true, + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + var conflict *ConflictError + if !errors.As(err, &conflict) || !strings.Contains(err.Error(), "returned before intent") { + t.Fatalf("initial Run() error = %v, want early-wait ConflictError", err) + } + if result.Phase != PhaseRebootIntentRecorded || harness.cluster.rebootRequests != 1 || cluster.waitCalls != 1 { + t.Fatalf("initial phase/requests/waits = %q/%d/%d, want %q/1/1", result.Phase, harness.cluster.rebootRequests, cluster.waitCalls, PhaseRebootIntentRecorded) + } + checkpoint, loadErr := LoadCheckpoint(outputDir) + if loadErr != nil { + t.Fatalf("load early-wait checkpoint: %v", loadErr) + } + if checkpoint.LastErrorCode != errorCodeConflict { + t.Fatalf("last error code = %q, want %q", checkpoint.LastErrorCode, errorCodeConflict) + } + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if !errors.As(err, &conflict) { + t.Fatalf("resumed Run() error = %v, want ConflictError", err) + } + if result.Phase != PhaseRebootIntentRecorded || harness.cluster.rebootRequests != 1 || cluster.waitCalls != 2 { + t.Fatalf("resumed phase/requests/waits = %q/%d/%d, want %q/1/2", result.Phase, harness.cluster.rebootRequests, cluster.waitCalls, PhaseRebootIntentRecorded) + } +} + +type rebootCoverageCluster struct { + *fakeClusterRotation + cancel context.CancelFunc + cancelAfter string + overrideRequest bool + requestOutcome EffectOutcome + requestErr error + failReadbackAfterRequest bool + failNextRebootReadback bool + waitWithoutCompletion bool + waitCalls int +} + +func (cluster *rebootCoverageCluster) PrepareReboot(ctx context.Context, guard RotationGuardReference, replacementKeyID string) (RebootPlan, error) { + plan, err := cluster.fakeClusterRotation.PrepareReboot(ctx, guard, replacementKeyID) + if err == nil && cluster.cancelAfter == "prepare-reboot" { + cluster.cancel() + } + return plan, err +} + +func (cluster *rebootCoverageCluster) RequestReboot(ctx context.Context, guard RotationGuardReference, intent RebootIntent) (EffectOutcome, error) { + if cluster.overrideRequest { + *cluster.events = append(*cluster.events, "cluster.request-reboot") + cluster.rebootRequests++ + return cluster.requestOutcome, cluster.requestErr + } + outcome, err := cluster.fakeClusterRotation.RequestReboot(ctx, guard, intent) + if cluster.failReadbackAfterRequest { + cluster.failReadbackAfterRequest = false + cluster.failNextRebootReadback = true + } + return outcome, err +} + +func (cluster *rebootCoverageCluster) ObserveReboot(ctx context.Context, guard RotationGuardReference, operationID string) (RebootObservation, error) { + if cluster.failNextRebootReadback { + cluster.failNextRebootReadback = false + *cluster.events = append(*cluster.events, "cluster.observe-reboot:error") + return RebootObservation{}, errors.New("reboot request readback failed") + } + return cluster.fakeClusterRotation.ObserveReboot(ctx, guard, operationID) +} + +func (cluster *rebootCoverageCluster) WaitForReboot(ctx context.Context, guard RotationGuardReference, intent RebootIntent) error { + cluster.waitCalls++ + if cluster.waitWithoutCompletion { + *cluster.events = append(*cluster.events, "cluster.wait-for-reboot") + return nil + } + return cluster.fakeClusterRotation.WaitForReboot(ctx, guard, intent) +} + +func (cluster *rebootCoverageCluster) WaitForPostRebootStable(ctx context.Context, guard RotationGuardReference, intent RebootIntent) error { + err := cluster.fakeClusterRotation.WaitForPostRebootStable(ctx, guard, intent) + if err == nil && cluster.cancelAfter == "post-reboot-stable" { + cluster.cancel() + } + return err +} + +func eventIndex(events []string, target string) int { + for index, event := range events { + if event == target { + return index + } + } + return -1 +} + +func countExactEvent(events []string, target string) int { + count := 0 + for _, event := range events { + if event == target { + count++ + } + } + return count +} + +func equalStrings(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} diff --git a/pkg/cmd/provisioning/rotation/rotation_guard_outcomes_test.go b/pkg/cmd/provisioning/rotation/rotation_guard_outcomes_test.go new file mode 100644 index 0000000000..5ad9e41b3a --- /dev/null +++ b/pkg/cmd/provisioning/rotation/rotation_guard_outcomes_test.go @@ -0,0 +1,392 @@ +package rotation + +import ( + "context" + "errors" + "testing" +) + +const otherGuardOperationID = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + +func TestAcquireRotationGuardRejectsPreexistingConflictsWithoutMutation(t *testing.T) { + for _, test := range []struct { + name string + status RotationGuardStatus + }{ + {name: "completed operation", status: RotationGuardCompleted}, + {name: "different owner", status: RotationGuardOwnedByOther}, + } { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeManual) + status := test.status + cluster := &guardOutcomeTestCluster{ + fakeClusterRotation: harness.cluster, + initialGuardObservation: &status, + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + + result, err := harness.orchestrator.Run(context.Background(), RunOptions{ + Provider: ProviderAWS, + PublicationMode: PublicationModeManual, + OutputDir: outputDir, + }) + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() error = %v, want ConflictError", err) + } + if result.Phase != PhasePreflightComplete || result.Complete { + t.Fatalf("Run() result = %#v, want stable preflight checkpoint", result) + } + if harness.cluster.guardAcquireCalls != 0 || harness.cluster.guardAcquireMutations != 0 { + t.Fatalf("guard acquisition = calls %d, mutations %d; want 0, 0", harness.cluster.guardAcquireCalls, harness.cluster.guardAcquireMutations) + } + requireGuardOutcomeCheckpointPhase(t, outputDir, PhasePreflightComplete) + }) + } +} + +func TestAcquireRotationGuardRetriesOnlyProvenUnappliedEffects(t *testing.T) { + mutationErr := errors.New("guard acquisition rejected") + readbackErr := errors.New("guard acquisition readback failed") + + for _, test := range []struct { + name string + plan guardMutationTestPlan + wantCause error + wantConflict bool + }{ + { + name: "mutation error", + plan: guardMutationTestPlan{outcome: EffectNotApplied, mutationErr: mutationErr}, + wantCause: mutationErr, + }, + { + name: "readback error", + plan: guardMutationTestPlan{outcome: EffectNotApplied, readbackErr: readbackErr}, + wantCause: readbackErr, + }, + { + name: "still not found", + plan: guardMutationTestPlan{outcome: EffectNotApplied}, + wantConflict: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeManual) + plan := test.plan + cluster := &guardOutcomeTestCluster{fakeClusterRotation: harness.cluster, acquirePlan: &plan} + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeManual, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + if test.wantConflict { + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() error = %v, want ConflictError", err) + } + } else if !errors.Is(err, test.wantCause) { + t.Fatalf("Run() error = %v, want cause %v", err, test.wantCause) + } + if result.Phase != PhasePreflightComplete || result.Complete { + t.Fatalf("Run() result = %#v, want stable preflight checkpoint", result) + } + if harness.cluster.guardAcquireCalls != 1 || harness.cluster.guardAcquireMutations != 0 { + t.Fatalf("guard acquisition = calls %d, mutations %d; want 1, 0", harness.cluster.guardAcquireCalls, harness.cluster.guardAcquireMutations) + } + requireGuardOutcomeCheckpointPhase(t, outputDir, PhasePreflightComplete) + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + requireOrchestratorPause(t, err, PhaseGuardAcquired, PauseForCurrentJWKS, ArtifactCurrentJWKS) + if result.Phase != PhaseGuardAcquired || result.Complete { + t.Fatalf("resumed Run() result = %#v, want guard-acquired pause", result) + } + if harness.cluster.guardAcquireCalls != 2 || harness.cluster.guardAcquireMutations != 1 { + t.Fatalf("guard acquisition after safe retry = calls %d, mutations %d; want 2, 1", harness.cluster.guardAcquireCalls, harness.cluster.guardAcquireMutations) + } + }) + } +} + +func TestAcquireRotationGuardReconcilesUncertainAppliedEffectsWithoutReplay(t *testing.T) { + for _, outcome := range []EffectOutcome{EffectUnknown, EffectSubmitted} { + t.Run(string(outcome), func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeManual) + plan := guardMutationTestPlan{ + outcome: outcome, + apply: true, + mutationErr: errors.New("guard acquisition response lost"), + readbackErr: errors.New("guard acquisition readback interrupted"), + } + cluster := &guardOutcomeTestCluster{fakeClusterRotation: harness.cluster, acquirePlan: &plan} + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeManual, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhasePreflightComplete || result.Complete { + t.Fatalf("Run() result = %#v, want stable preflight checkpoint", result) + } + if harness.cluster.guardAcquireCalls != 1 || harness.cluster.guardAcquireMutations != 1 { + t.Fatalf("guard acquisition = calls %d, mutations %d; want 1, 1", harness.cluster.guardAcquireCalls, harness.cluster.guardAcquireMutations) + } + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + requireOrchestratorPause(t, err, PhaseGuardAcquired, PauseForCurrentJWKS, ArtifactCurrentJWKS) + if result.Phase != PhaseGuardAcquired || result.Complete { + t.Fatalf("resumed Run() result = %#v, want guard-acquired pause", result) + } + if harness.cluster.guardAcquireCalls != 1 || harness.cluster.guardAcquireMutations != 1 { + t.Fatalf("guard acquisition was replayed: calls %d, mutations %d; want 1, 1", harness.cluster.guardAcquireCalls, harness.cluster.guardAcquireMutations) + } + }) + } +} + +func TestReleaseRotationGuardRetriesOnlyProvenUnappliedEffects(t *testing.T) { + mutationErr := errors.New("guard release rejected") + readbackErr := errors.New("guard release readback failed") + + for _, test := range []struct { + name string + plan guardMutationTestPlan + wantCause error + wantConflict bool + }{ + { + name: "mutation error", + plan: guardMutationTestPlan{outcome: EffectNotApplied, mutationErr: mutationErr}, + wantCause: mutationErr, + }, + { + name: "readback error", + plan: guardMutationTestPlan{outcome: EffectNotApplied, readbackErr: readbackErr}, + wantCause: readbackErr, + }, + { + name: "still held", + plan: guardMutationTestPlan{outcome: EffectNotApplied}, + wantConflict: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + plan := test.plan + cluster := &guardOutcomeTestCluster{fakeClusterRotation: harness.cluster, releasePlan: &plan} + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + if test.wantConflict { + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Run() error = %v, want ConflictError", err) + } + } else if !errors.Is(err, test.wantCause) { + t.Fatalf("Run() error = %v, want cause %v", err, test.wantCause) + } + if result.Phase != PhaseGuardReleaseRecorded || result.Complete { + t.Fatalf("Run() result = %#v, want stable guard-release checkpoint", result) + } + if harness.cluster.guardReleaseCalls != 1 || harness.cluster.guardReleaseMutations != 0 { + t.Fatalf("guard release = calls %d, mutations %d; want 1, 0", harness.cluster.guardReleaseCalls, harness.cluster.guardReleaseMutations) + } + requireGuardOutcomeCheckpointPhase(t, outputDir, PhaseGuardReleaseRecorded) + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if err != nil || !result.Complete || result.Phase != PhaseComplete { + t.Fatalf("resumed Run() result = %#v, error = %v; want complete", result, err) + } + if harness.cluster.guardReleaseCalls != 2 || harness.cluster.guardReleaseMutations != 1 { + t.Fatalf("guard release after safe retry = calls %d, mutations %d; want 2, 1", harness.cluster.guardReleaseCalls, harness.cluster.guardReleaseMutations) + } + }) + } +} + +func TestReleaseRotationGuardReconcilesUncertainAppliedEffectsWithoutReplay(t *testing.T) { + for _, outcome := range []EffectOutcome{EffectUnknown, EffectSubmitted} { + t.Run(string(outcome), func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + plan := guardMutationTestPlan{ + outcome: outcome, + apply: true, + mutationErr: errors.New("guard release response lost"), + readbackErr: errors.New("guard release readback interrupted"), + } + cluster := &guardOutcomeTestCluster{fakeClusterRotation: harness.cluster, releasePlan: &plan} + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhaseGuardReleaseRecorded || result.Complete { + t.Fatalf("Run() result = %#v, want stable guard-release checkpoint", result) + } + if harness.cluster.guardReleaseCalls != 1 || harness.cluster.guardReleaseMutations != 1 { + t.Fatalf("guard release = calls %d, mutations %d; want 1, 1", harness.cluster.guardReleaseCalls, harness.cluster.guardReleaseMutations) + } + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if err != nil || !result.Complete || result.Phase != PhaseComplete { + t.Fatalf("resumed Run() result = %#v, error = %v; want complete", result, err) + } + if harness.cluster.guardReleaseCalls != 1 || harness.cluster.guardReleaseMutations != 1 { + t.Fatalf("guard release was replayed: calls %d, mutations %d; want 1, 1", harness.cluster.guardReleaseCalls, harness.cluster.guardReleaseMutations) + } + }) + } +} + +func TestReleaseCheckpointRejectsLostOrConflictingGuardWithoutAnotherMutation(t *testing.T) { + for _, test := range []struct { + name string + status RotationGuardStatus + }{ + {name: "completed operation", status: RotationGuardCompleted}, + {name: "different owner", status: RotationGuardOwnedByOther}, + {name: "guard not found", status: RotationGuardNotFound}, + } { + t.Run(test.name, func(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + plan := guardMutationTestPlan{outcome: EffectNotApplied, mutationErr: errors.New("stop at release checkpoint")} + cluster := &guardOutcomeTestCluster{fakeClusterRotation: harness.cluster, releasePlan: &plan} + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + if err == nil || result.Phase != PhaseGuardReleaseRecorded { + t.Fatalf("initial Run() result = %#v, error = %v; want guard-release checkpoint", result, err) + } + checkpoint := requireGuardOutcomeCheckpointPhase(t, outputDir, PhaseGuardReleaseRecorded) + switch test.status { + case RotationGuardCompleted: + harness.cluster.guardReference = nil + harness.cluster.completedGuards = map[string]struct{}{checkpoint.RotationGuard.OperationID: {}} + case RotationGuardOwnedByOther: + other := *checkpoint.RotationGuard + other.OperationID = otherGuardOperationID + harness.cluster.guardReference = &other + case RotationGuardNotFound: + harness.cluster.guardReference = nil + } + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if test.status == RotationGuardCompleted { + if err != nil || !result.Complete || result.Phase != PhaseComplete { + t.Fatalf("resumed Run() result = %#v, error = %v; want completed observation to finish", result, err) + } + } else { + var conflict *ConflictError + if !errors.As(err, &conflict) { + t.Fatalf("resumed Run() error = %v, want ConflictError", err) + } + if result.Phase != PhaseGuardReleaseRecorded || result.Complete { + t.Fatalf("resumed Run() result = %#v, want stable guard-release checkpoint", result) + } + requireGuardOutcomeCheckpointPhase(t, outputDir, PhaseGuardReleaseRecorded) + } + if harness.cluster.guardReleaseCalls != 1 || harness.cluster.guardReleaseMutations != 0 { + t.Fatalf("guard release after reconciliation = calls %d, mutations %d; want 1, 0", harness.cluster.guardReleaseCalls, harness.cluster.guardReleaseMutations) + } + }) + } +} + +type guardMutationTestPlan struct { + outcome EffectOutcome + apply bool + mutationErr error + readbackErr error +} + +type guardOutcomeTestCluster struct { + *fakeClusterRotation + initialGuardObservation *RotationGuardStatus + acquirePlan *guardMutationTestPlan + releasePlan *guardMutationTestPlan + nextGuardObservationErr error +} + +func (c *guardOutcomeTestCluster) ObserveRotationGuard(ctx context.Context, reference RotationGuardReference) (RotationGuardObservation, error) { + if c.nextGuardObservationErr != nil { + *c.events = append(*c.events, "cluster.observe-rotation-guard") + err := c.nextGuardObservationErr + c.nextGuardObservationErr = nil + return RotationGuardObservation{}, err + } + if c.initialGuardObservation != nil { + *c.events = append(*c.events, "cluster.observe-rotation-guard") + status := *c.initialGuardObservation + c.initialGuardObservation = nil + observation := RotationGuardObservation{Status: status} + switch status { + case RotationGuardHeld, RotationGuardCompleted: + observation.OperationID = reference.OperationID + case RotationGuardOwnedByOther: + observation.OperationID = otherGuardOperationID + } + return observation, nil + } + return c.fakeClusterRotation.ObserveRotationGuard(ctx, reference) +} + +func (c *guardOutcomeTestCluster) AcquireRotationGuard(ctx context.Context, reference RotationGuardReference) (EffectOutcome, error) { + if c.acquirePlan == nil { + return c.fakeClusterRotation.AcquireRotationGuard(ctx, reference) + } + plan := *c.acquirePlan + c.acquirePlan = nil + if plan.apply { + _, _ = c.fakeClusterRotation.AcquireRotationGuard(ctx, reference) + } else { + *c.events = append(*c.events, "cluster.acquire-rotation-guard") + c.guardAcquireCalls++ + } + c.nextGuardObservationErr = plan.readbackErr + return plan.outcome, plan.mutationErr +} + +func (c *guardOutcomeTestCluster) ReleaseRotationGuard(ctx context.Context, reference RotationGuardReference) (EffectOutcome, error) { + if c.releasePlan == nil { + return c.fakeClusterRotation.ReleaseRotationGuard(ctx, reference) + } + plan := *c.releasePlan + c.releasePlan = nil + if plan.apply { + _, _ = c.fakeClusterRotation.ReleaseRotationGuard(ctx, reference) + } else { + *c.events = append(*c.events, "cluster.release-rotation-guard") + c.guardReleaseCalls++ + } + c.nextGuardObservationErr = plan.readbackErr + return plan.outcome, plan.mutationErr +} + +func requireGuardOutcomeCheckpointPhase(t *testing.T, outputDir string, want Phase) Checkpoint { + t.Helper() + checkpoint, err := LoadCheckpoint(outputDir) + if err != nil { + t.Fatalf("LoadCheckpoint() returned unexpected error: %v", err) + } + if checkpoint.Phase != want { + t.Fatalf("checkpoint phase = %q, want %q", checkpoint.Phase, want) + } + return checkpoint +} From 17b4511e17c1503a032a122831ac1a96837f096e Mon Sep 17 00:00:00 2001 From: Bruno Andrade Date: Tue, 8 Sep 2026 16:22:12 -0300 Subject: [PATCH 4/6] docs: Clarify future rotation command forms --- docs/ccoctl-rotate-signing-key.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/ccoctl-rotate-signing-key.md b/docs/ccoctl-rotate-signing-key.md index 50a794535b..381a29f06b 100644 --- a/docs/ccoctl-rotate-signing-key.md +++ b/docs/ccoctl-rotate-signing-key.md @@ -6,10 +6,16 @@ ## Scope -The command is invoked externally for exactly one provider and one cluster: +The initial AWS provider command is invoked externally for exactly one provider and one cluster: ```text ccoctl aws rotate-signing-key [options] +``` + +Future Azure and GCP provider adapters are expected to use the equivalent contract forms below; +these commands are not registered by this foundation: + +```text ccoctl azure rotate-signing-key [options] ccoctl gcp rotate-signing-key [options] ``` From a3b0827412979c5095ff5517860c39ae4205ae0a Mon Sep 17 00:00:00 2001 From: Bruno Andrade Date: Tue, 8 Sep 2026 15:21:24 -0300 Subject: [PATCH 5/6] ccoctl: Add AWS signing-key rotation Register the AWS direct-publication workflow for ccoctl-managed S3 issuers and add the concrete cluster adapter for guarded signer replacement, rollout verification, and crash-safe node reboot reconciliation. Preserve S3 object ownership and tags across conditional JWKS updates, validate supported MachineConfigPool topology before mutation, and document the resumable command and required permissions. Assisted-by: GPT-5 Signed-off-by: Bruno Andrade --- docs/ccoctl-rotate-signing-key.md | 31 +- docs/ccoctl.md | 39 +- docs/rotate-oidc-key.md | 36 +- pkg/aws/client.go | 5 + pkg/aws/mock/client_generated.go | 20 + pkg/cmd/provisioning/aws/aws.go | 1 + .../provisioning/aws/rotate_signing_key.go | 144 +++ .../aws/rotate_signing_key_test.go | 217 ++++ pkg/cmd/provisioning/aws/rotation_s3.go | 470 +++++++++ pkg/cmd/provisioning/aws/rotation_s3_test.go | 470 +++++++++ .../rotation/kubeadapter/adapter.go | 239 +++++ .../rotation/kubeadapter/guard.go | 264 +++++ .../rotation/kubeadapter/metadata.go | 75 ++ .../kubeadapter/metadata_guard_signer_test.go | 227 +++++ .../rotation/kubeadapter/preflight.go | 365 +++++++ .../rotation/kubeadapter/reboot.go | 923 ++++++++++++++++++ .../kubeadapter/reboot_preflight_test.go | 740 ++++++++++++++ .../rotation/kubeadapter/signer.go | 198 ++++ .../rotation/kubeadapter/test_helpers_test.go | 253 +++++ pkg/cmd/provisioning/rotation/orchestrator.go | 23 +- .../rotation/orchestrator_safety_test.go | 2 +- .../rotation/orchestrator_test.go | 2 + .../rotation/orchestrator_types.go | 23 +- .../rotation/reboot_cancellation_test.go | 95 +- 24 files changed, 4821 insertions(+), 41 deletions(-) create mode 100644 pkg/cmd/provisioning/aws/rotate_signing_key.go create mode 100644 pkg/cmd/provisioning/aws/rotate_signing_key_test.go create mode 100644 pkg/cmd/provisioning/aws/rotation_s3.go create mode 100644 pkg/cmd/provisioning/aws/rotation_s3_test.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/adapter.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/guard.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/metadata.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/metadata_guard_signer_test.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/preflight.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/reboot.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/signer.go create mode 100644 pkg/cmd/provisioning/rotation/kubeadapter/test_helpers_test.go diff --git a/docs/ccoctl-rotate-signing-key.md b/docs/ccoctl-rotate-signing-key.md index 381a29f06b..bea2fdf786 100644 --- a/docs/ccoctl-rotate-signing-key.md +++ b/docs/ccoctl-rotate-signing-key.md @@ -1,8 +1,9 @@ # `ccoctl` signing-key rotation command contract -> **Status:** The shared phase engine and command factory implement this provider-neutral contract, -> but the provider commands and their concrete cluster/cloud adapters are not registered yet. Until -> those integrations are available, use the [manual rotation procedure](rotate-oidc-key.md). +> **Status:** The shared phase engine and command factory implement this provider-neutral contract. +> AWS direct publication is registered for the standard ccoctl-managed S3 issuer layout. Azure, +> GCP, manual publication, and custom issuer layouts still require the +> [manual rotation procedure](rotate-oidc-key.md). ## Scope @@ -67,6 +68,12 @@ The command contract requires the following capabilities: Secret with UID and resource version preconditions to trigger regeneration, acquire and complete the cluster-durable signer-rotation guard, and reboot and observe the applicable worker and control-plane machine config pools. +- Every cluster Node must be stably managed by the Machine Config Operator. A + provider command must fail before signer replacement rather than omit an + unmanaged Node from the reboot evidence. Nodes in a custom MachineConfigPool + are supported only when that pool inherits exactly one of the standard + master or worker MachineConfigs. A separate role pool such as `arbiter` is + rejected during preflight until a dedicated reboot target is supported. - Enough cloud permission to locate and read the current JWKS and to publish both the combined and final JWKS to the supported target. - A durable working directory for public artifacts and resumable, non-secret checkpoint state. @@ -189,10 +196,12 @@ The implementation must durably record these exact schema-version-1 phases in or and each target node's baseline boot ID are durably recorded before requesting a reboot. The shared engine derives the operation ID deterministically from the cluster identity and replacement key ID so retries using another working directory converge even when their observed - targets or boot-ID baselines differ. The first reboot request atomically creates a - cluster-durable canonical record containing its exact targets and baselines. Later workspaces - adopt that canonical record before advancing, and it remains observable for the lifetime of - resumable operation state. + targets or boot-ID baselines differ. The first reboot request durably creates a + cluster-canonical record containing its exact targets and baselines before it triggers any + target. The request then reconciles each target idempotently. Later workspaces adopt that + canonical record before advancing, and an interrupted request is re-entered while it is in + progress to finish only targets that do not already record the same operation ID. The canonical + record remains observable for the lifetime of resumable operation state. 12. **`nodes-rebooted`:** reboot completion is confirmed for every node recorded in the immutable reboot intent. 13. **`post-reboot-stable`:** the cluster reports stability after the node reboots. @@ -246,9 +255,11 @@ without requiring a publicly reachable JWKS endpoint. key merely because the previous run ended. - If a provider write has an unknown outcome, retry must read or otherwise reconcile the target before writing again. -- The reboot intent must be checkpointed before the disruptive request. Resume must reconcile node - boot IDs against the cluster-canonical baselines and reuse the same operation ID; it must not - submit a second pool reboot merely because the previous run ended or its outcome is unknown. +- The reboot intent must be checkpointed before the disruptive request. The cluster-canonical intent + must then be durable before any target is triggered. Resume must reconcile node boot IDs against + those canonical baselines and reuse the same operation ID. It may re-enter the request to finish + targets left incomplete by an interruption, but must not trigger a target that already records + that operation ID merely because the previous run ended or its outcome is unknown. - Reboot planning, observation, reconciliation, and waits are read-only. The reboot request is the only operation allowed to create the canonical reboot record or trigger the disruptive action. - Repeated execution must not duplicate keys in the combined JWKS, regress to an earlier artifact, diff --git a/docs/ccoctl.md b/docs/ccoctl.md index 6540a667b7..805e43b5f5 100644 --- a/docs/ccoctl.md +++ b/docs/ccoctl.md @@ -2,13 +2,14 @@ The `ccoctl` tool provides various commands to assist with the creating and maintenance of cloud credentials from outside the cluster (necessary when CCO is put in "Manual" mode). -The planned provider-specific signing-key rotation commands are defined by the -[`rotate-signing-key` command contract](ccoctl-rotate-signing-key.md). Their shared engine exists, -but the provider adapters and commands are not registered yet; use the -[manual rotation procedure](rotate-oidc-key.md) in the meantime. +The provider-specific signing-key rotation workflow is defined by the +[`rotate-signing-key` command contract](ccoctl-rotate-signing-key.md). AWS direct publication for +the standard ccoctl-managed S3 issuer layout is available as a command. Use the +[manual rotation procedure](rotate-oidc-key.md) for other layouts and providers. - [AWS](#aws) - [Global flags](#global-flags) + - [Rotating the signing key](#rotating-the-signing-key) - [Creating RSA keys](#creating-rsa-keys) - [Creating OpenID Connect Provider](#creating-openid-connect-provider) - [Creating IAM Roles](#creating-iam-roles) @@ -49,6 +50,36 @@ By default, the tool will output to the directory the command(s) were run in. To Commands which would otherwise make AWS API calls can be passed the `--dry-run` flag to have `ccoctl` place JSON files on the local filesystem instead of creating/modifying any AWS resources. These JSON files can be reviewed/modified and then applied with the `aws` CLI tool (using the `--cli-input-json` parameters). +### Rotating the signing key + +To rotate the signer and publish the overlapping JWKS directly to the standard +ccoctl-managed S3 issuer, run: + +```bash +$ ccoctl aws rotate-signing-key \ + --name= \ + --region= \ + --kubeconfig=/absolute/path/to/kubeconfig \ + --output-dir=/path/to/rotation-state +``` + +The command derives the issuer bucket as `-oidc`, updates its `keys.json` +object, and uses `name` to verify ccoctl resource ownership. It loads AWS +credentials from the standard AWS SDK credential chain and supports direct +publication only. Do not run it while another process is changing the object's +tags: S3 conditionally protects the object content by ETag, while tag and +version drift can only be checked immediately before the write. Use `--resume` +with the same arguments and output directory to continue a previously +checkpointed rotation. The AWS identity needs `s3:GetObject`, +`s3:GetObjectTagging` (and `s3:GetObjectVersionTagging` for a versioned object), +`s3:PutObject`, and `s3:PutObjectTagging` on `keys.json`. + +All cluster Nodes must be stably managed by the Machine Config Operator. The +command fails closed during preflight rather than excluding an unmanaged Node +from the reboot evidence. Custom MachineConfigPools must inherit exactly one of +the standard master or worker MachineConfigs; separate role pools such as +`arbiter` are rejected before rotation begins. + ### Creating RSA keys To generate keys for use when setting up the cluster's OpenID Connect provider, run diff --git a/docs/rotate-oidc-key.md b/docs/rotate-oidc-key.md index cfd8b5d9b7..0ac27db400 100644 --- a/docs/rotate-oidc-key.md +++ b/docs/rotate-oidc-key.md @@ -3,9 +3,10 @@ ## Overview When OpenShift is configured to use temporary credentials (AZWI, STS, WIF) to authenticate with the cloud platform api, special care must be taken when rotating the bound service account signer keys in order to reduce authentication failures. This can be accomplished by adding the new public key to the existing issuer file immediately after the cluster generates it. Once the cluster has fully updated to the new key all other keys can be removed. -This page describes the current manual procedure. The provider-neutral workflow is defined in the -[`ccoctl rotate-signing-key` command contract](ccoctl-rotate-signing-key.md), but its provider -adapters and commands are not registered yet. +This page describes the manual procedure. The provider-neutral workflow is defined in the +[`ccoctl rotate-signing-key` command contract](ccoctl-rotate-signing-key.md). AWS direct +publication for the standard ccoctl-managed S3 issuer layout is available through `ccoctl`; the +manual procedure remains necessary for other layouts and providers. This manual procedure assumes an exclusive maintenance window in which no other user, automation, or controller deletes `next-bound-service-account-signing-key`. Stop if that exclusivity cannot be @@ -15,9 +16,30 @@ produced an entry, so concurrent rotations cannot be disambiguated safely by thi The provider upload examples below are legacy unconditional writes. Ensure exclusive publisher access, verify that the remote JWKS still equals the expected predecessor immediately before each upload, and read it back to compare with the exact uploaded file afterward. Stop if either -comparison fails or if the provider cannot be protected from another writer. The provider adapters -described by the command contract must replace this operational assumption with conditional writes -and exact readback. +comparison fails or if the provider cannot be protected from another writer. The AWS command +replaces this operational assumption with conditional writes and exact readback; other provider +adapters must do likewise. + +## AWS standard S3 issuer + +For an issuer created with the standard ccoctl-managed S3 layout, prefer the +checkpointed direct-publication command: + +```bash +$ ccoctl aws rotate-signing-key \ + --name= \ + --region= \ + --kubeconfig=/absolute/path/to/kubeconfig \ + --output-dir=/path/to/rotation-state +``` + +The command derives `-oidc` as the bucket, publishes `keys.json`, and +checks ownership using `name`. It uses the standard AWS SDK credential chain. +Manual publication mode and custom issuer layouts are not supported by this +command. Do not change the object's tags concurrently: the S3 write is +conditional on the content ETag, while tag and version drift are checked +immediately before it. If an interrupted run has a checkpoint in the output +directory, add `--resume` and retain the same provider arguments. ## Process @@ -90,7 +112,7 @@ and exact readback. | jq -er '.oidc.jwksJson' > "${TEMPDIR}/jwks.current.download.json" ``` - Perform this basic structural check before continuing. The forthcoming command also performs + Perform this basic structural check before continuing. The checkpointed AWS command also performs strict RSA key, key ID, algorithm, purpose, duplicate, and signer-baseline validation that is not reproduced by this legacy shell procedure. diff --git a/pkg/aws/client.go b/pkg/aws/client.go index 1d0c717fb1..174f02f8ba 100644 --- a/pkg/aws/client.go +++ b/pkg/aws/client.go @@ -71,6 +71,7 @@ type Client interface { PutBucketTagging(context.Context, *s3.PutBucketTaggingInput, ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) GetBucketTagging(context.Context, *s3.GetBucketTaggingInput, ...func(*s3.Options)) (*s3.GetBucketTaggingOutput, error) DeleteBucket(context.Context, *s3.DeleteBucketInput, ...func(*s3.Options)) (*s3.DeleteBucketOutput, error) + GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) ListObjectsV2(context.Context, *s3.ListObjectsV2Input, ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) GetObjectTagging(context.Context, *s3.GetObjectTaggingInput, ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) @@ -221,6 +222,10 @@ func (c *awsClient) DeleteBucket(ctx context.Context, input *s3.DeleteBucketInpu return c.s3Client.DeleteBucket(ctx, input, opts...) } +func (c *awsClient) GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + return c.s3Client.GetObject(ctx, input, opts...) +} + func (c *awsClient) PutObject(ctx context.Context, input *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) { return c.s3Client.PutObject(ctx, input, opts...) } diff --git a/pkg/aws/mock/client_generated.go b/pkg/aws/mock/client_generated.go index 781448b317..8815f3d407 100644 --- a/pkg/aws/mock/client_generated.go +++ b/pkg/aws/mock/client_generated.go @@ -437,6 +437,26 @@ func (mr *MockClientMockRecorder) GetDistribution(arg0, arg1 interface{}, arg2 . return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDistribution", reflect.TypeOf((*MockClient)(nil).GetDistribution), varargs...) } +// GetObject mocks base method. +func (m *MockClient) GetObject(arg0 context.Context, arg1 *s3.GetObjectInput, arg2 ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetObject", varargs...) + ret0, _ := ret[0].(*s3.GetObjectOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObject indicates an expected call of GetObject. +func (mr *MockClientMockRecorder) GetObject(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObject", reflect.TypeOf((*MockClient)(nil).GetObject), varargs...) +} + // GetObjectTagging mocks base method. func (m *MockClient) GetObjectTagging(arg0 context.Context, arg1 *s3.GetObjectTaggingInput, arg2 ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { m.ctrl.T.Helper() diff --git a/pkg/cmd/provisioning/aws/aws.go b/pkg/cmd/provisioning/aws/aws.go index 35d33f09bc..e793579e8d 100644 --- a/pkg/cmd/provisioning/aws/aws.go +++ b/pkg/cmd/provisioning/aws/aws.go @@ -37,6 +37,7 @@ func NewAWSCmd() *cobra.Command { createCmd.AddCommand(NewCreateIAMRolesCmd()) createCmd.AddCommand(NewCreateAllCmd()) createCmd.AddCommand(NewDeleteCmd()) + createCmd.AddCommand(NewRotateSigningKeyCmd()) return createCmd } diff --git a/pkg/cmd/provisioning/aws/rotate_signing_key.go b/pkg/cmd/provisioning/aws/rotate_signing_key.go new file mode 100644 index 0000000000..05801ffc02 --- /dev/null +++ b/pkg/cmd/provisioning/aws/rotate_signing_key.go @@ -0,0 +1,144 @@ +package aws + +import ( + "context" + "fmt" + "strings" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/spf13/cobra" + + awsclient "github.com/openshift/cloud-credential-operator/pkg/aws" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation/kubeadapter" +) + +type rotateSigningKeyOptions struct { + name string + region string +} + +type awsRotationDependencies struct { + loadAWSConfig func(context.Context, string) (awssdk.Config, error) + newAWSClient func(awssdk.Config) (awsclient.Client, error) + newCluster func(string) (rotation.ClusterRotation, error) + newTarget func(string, string) (rotation.TargetResolver, error) + newPublisher func(awsclient.Client, string, string, string) (rotation.ConditionalJWKSBackend, error) + newOrchestrator func(rotation.ClusterRotation, rotation.TargetResolver, rotation.ConditionalJWKSBackend) rotation.Runner +} + +// NewRotateSigningKeyCmd returns the AWS direct-publication signing-key +// rotation command for the standard ccoctl-managed S3 issuer layout. +func NewRotateSigningKeyCmd() *cobra.Command { + return newRotateSigningKeyCmd(defaultAWSRotationDependencies()) +} + +func newRotateSigningKeyCmd(dependencies awsRotationDependencies) *cobra.Command { + options := rotateSigningKeyOptions{} + command := rotation.NewCommand(rotation.ProviderAWS, newAWSRotationRunnerFactory(&options, dependencies)) + + flags := command.Flags() + flags.StringVar(&options.name, "name", "", "Name used to identify and own the AWS resources") + flags.StringVar(&options.region, "region", "", "AWS region containing the OIDC issuer bucket") + mustMarkAWSRotationFlagRequired(command, "name") + mustMarkAWSRotationFlagRequired(command, "region") + + return command +} + +func newAWSRotationRunnerFactory(options *rotateSigningKeyOptions, dependencies awsRotationDependencies) rotation.RunnerFactory { + return func(ctx context.Context, kubeconfig string, runOptions rotation.RunOptions) (rotation.RunnerSetup, error) { + if runOptions.PublicationMode != rotation.PublicationModeDirect { + return rotation.RunnerSetup{}, fmt.Errorf("AWS signing-key rotation supports only direct publication") + } + if options == nil { + return rotation.RunnerSetup{}, fmt.Errorf("AWS signing-key rotation options must not be nil") + } + if strings.TrimSpace(options.name) == "" || strings.TrimSpace(options.name) != options.name { + return rotation.RunnerSetup{}, fmt.Errorf("AWS resource name must not be empty or contain surrounding whitespace") + } + if strings.TrimSpace(options.region) == "" || strings.TrimSpace(options.region) != options.region { + return rotation.RunnerSetup{}, fmt.Errorf("AWS region must not be empty or contain surrounding whitespace") + } + if err := validateAWSRotationDependencies(dependencies); err != nil { + return rotation.RunnerSetup{}, err + } + + bucket := options.name + "-oidc" + target, err := dependencies.newTarget(options.region, bucket) + if err != nil { + return rotation.RunnerSetup{}, fmt.Errorf("construct AWS S3 rotation target: %w", err) + } + cluster, err := dependencies.newCluster(kubeconfig) + if err != nil { + return rotation.RunnerSetup{}, fmt.Errorf("construct cluster rotation adapter: %w", err) + } + cfg, err := dependencies.loadAWSConfig(ctx, options.region) + if err != nil { + return rotation.RunnerSetup{}, fmt.Errorf("load AWS configuration: %w", err) + } + client, err := dependencies.newAWSClient(cfg) + if err != nil { + return rotation.RunnerSetup{}, fmt.Errorf("construct AWS client: %w", err) + } + publisher, err := dependencies.newPublisher(client, options.region, bucket, options.name) + if err != nil { + return rotation.RunnerSetup{}, fmt.Errorf("construct AWS S3 JWKS publisher: %w", err) + } + runner := dependencies.newOrchestrator(cluster, target, publisher) + if runner == nil { + return rotation.RunnerSetup{}, fmt.Errorf("construct AWS signing-key rotation runner: returned nil runner") + } + + return rotation.RunnerSetup{Runner: runner}, nil + } +} + +func defaultAWSRotationDependencies() awsRotationDependencies { + return awsRotationDependencies{ + loadAWSConfig: func(ctx context.Context, region string) (awssdk.Config, error) { + return config.LoadDefaultConfig(ctx, config.WithRegion(region)) + }, + newAWSClient: func(cfg awssdk.Config) (awsclient.Client, error) { + return awsclient.NewClientFromConfig(cfg, "") + }, + newCluster: func(kubeconfig string) (rotation.ClusterRotation, error) { + return kubeadapter.New(kubeconfig) + }, + newTarget: func(region, bucket string) (rotation.TargetResolver, error) { + return NewS3TargetResolver(region, bucket) + }, + newPublisher: func(client awsclient.Client, region, bucket, ownerName string) (rotation.ConditionalJWKSBackend, error) { + return NewS3JWKSBackend(client, region, bucket, ownerName) + }, + newOrchestrator: func(cluster rotation.ClusterRotation, target rotation.TargetResolver, publisher rotation.ConditionalJWKSBackend) rotation.Runner { + return &rotation.Orchestrator{Cluster: cluster, Target: target, Publisher: publisher} + }, + } +} + +func validateAWSRotationDependencies(dependencies awsRotationDependencies) error { + switch { + case dependencies.loadAWSConfig == nil: + return fmt.Errorf("AWS configuration loader must not be nil") + case dependencies.newAWSClient == nil: + return fmt.Errorf("AWS client constructor must not be nil") + case dependencies.newCluster == nil: + return fmt.Errorf("cluster rotation adapter constructor must not be nil") + case dependencies.newTarget == nil: + return fmt.Errorf("AWS S3 target constructor must not be nil") + case dependencies.newPublisher == nil: + return fmt.Errorf("AWS S3 publisher constructor must not be nil") + case dependencies.newOrchestrator == nil: + return fmt.Errorf("rotation orchestrator constructor must not be nil") + default: + return nil + } +} + +func mustMarkAWSRotationFlagRequired(command *cobra.Command, name string) { + if err := command.MarkFlagRequired(name); err != nil { + panic(err) + } +} diff --git a/pkg/cmd/provisioning/aws/rotate_signing_key_test.go b/pkg/cmd/provisioning/aws/rotate_signing_key_test.go new file mode 100644 index 0000000000..c517d35a89 --- /dev/null +++ b/pkg/cmd/provisioning/aws/rotate_signing_key_test.go @@ -0,0 +1,217 @@ +package aws + +import ( + "context" + "io" + "reflect" + "strings" + "testing" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + + awsclient "github.com/openshift/cloud-credential-operator/pkg/aws" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +func TestNewAWSCmdRegistersRotateSigningKey(t *testing.T) { + t.Parallel() + + command := NewAWSCmd() + for _, child := range command.Commands() { + if child.Name() == "rotate-signing-key" { + return + } + } + t.Fatal("AWS command does not register rotate-signing-key") +} + +func TestAWSRotateSigningKeyCommandWiresStandardS3DirectRunner(t *testing.T) { + t.Parallel() + + type contextKey string + const key contextKey = "aws-rotation-command" + ctx := context.WithValue(context.Background(), key, "context-value") + + cluster := &awsRotationTestCluster{} + target := &awsRotationTestTarget{} + publisher := &awsRotationTestPublisher{} + client := &awsRotationTestClient{} + runner := &awsRotationTestRunner{result: rotation.RunResult{Phase: rotation.PhaseComplete, Complete: true}} + + var clusterKubeconfig string + var configContext context.Context + var configRegion string + var clientConfig awssdk.Config + var targetRegion, targetBucket string + var publisherRegion, publisherBucket, publisherOwner string + var runnerCluster rotation.ClusterRotation + var runnerTarget rotation.TargetResolver + var runnerPublisher rotation.ConditionalJWKSBackend + + dependencies := awsRotationDependencies{ + loadAWSConfig: func(receivedContext context.Context, region string) (awssdk.Config, error) { + configContext = receivedContext + configRegion = region + return awssdk.Config{Region: region}, nil + }, + newAWSClient: func(cfg awssdk.Config) (awsclient.Client, error) { + clientConfig = cfg + return client, nil + }, + newCluster: func(kubeconfig string) (rotation.ClusterRotation, error) { + clusterKubeconfig = kubeconfig + return cluster, nil + }, + newTarget: func(region, bucket string) (rotation.TargetResolver, error) { + targetRegion = region + targetBucket = bucket + return target, nil + }, + newPublisher: func(receivedClient awsclient.Client, region, bucket, owner string) (rotation.ConditionalJWKSBackend, error) { + if receivedClient != client { + t.Fatal("publisher constructor received a different AWS client") + } + publisherRegion = region + publisherBucket = bucket + publisherOwner = owner + return publisher, nil + }, + newOrchestrator: func(receivedCluster rotation.ClusterRotation, receivedTarget rotation.TargetResolver, receivedPublisher rotation.ConditionalJWKSBackend) rotation.Runner { + runnerCluster = receivedCluster + runnerTarget = receivedTarget + runnerPublisher = receivedPublisher + return runner + }, + } + + command := newRotateSigningKeyCmd(dependencies) + command.SetOut(io.Discard) + command.SetErr(io.Discard) + command.SetArgs([]string{ + "--name", "example-cluster", + "--region", "us-east-1", + "--kubeconfig", "/tmp/example.kubeconfig", + "--output-dir", "/tmp/rotation-output", + "--resume", + }) + + if err := command.ExecuteContext(ctx); err != nil { + t.Fatalf("ExecuteContext() returned unexpected error: %v", err) + } + + if clusterKubeconfig != "/tmp/example.kubeconfig" { + t.Fatalf("cluster kubeconfig = %q, want /tmp/example.kubeconfig", clusterKubeconfig) + } + if configContext != ctx || configContext.Value(key) != "context-value" { + t.Fatal("AWS configuration loader did not receive the command context") + } + if configRegion != "us-east-1" || clientConfig.Region != "us-east-1" { + t.Fatalf("AWS configuration regions = %q and %q, want us-east-1", configRegion, clientConfig.Region) + } + if targetRegion != "us-east-1" || targetBucket != "example-cluster-oidc" { + t.Fatalf("target arguments = (%q, %q), want (us-east-1, example-cluster-oidc)", targetRegion, targetBucket) + } + if publisherRegion != "us-east-1" || publisherBucket != "example-cluster-oidc" || publisherOwner != "example-cluster" { + t.Fatalf("publisher arguments = (%q, %q, %q), want (us-east-1, example-cluster-oidc, example-cluster)", publisherRegion, publisherBucket, publisherOwner) + } + if runnerCluster != cluster || runnerTarget != target || runnerPublisher != publisher { + t.Fatal("orchestrator constructor did not receive the constructed adapters") + } + wantOptions := rotation.RunOptions{ + Provider: rotation.ProviderAWS, + PublicationMode: rotation.PublicationModeDirect, + OutputDir: "/tmp/rotation-output", + Resume: true, + } + if !reflect.DeepEqual(runner.options, wantOptions) { + t.Fatalf("runner options = %+v, want %+v", runner.options, wantOptions) + } + if runner.ctx != ctx || runner.calls != 1 { + t.Fatalf("runner received context %v and %d calls, want command context and 1 call", runner.ctx, runner.calls) + } +} + +func TestAWSRotateSigningKeyCommandRequiresProviderFlags(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + wantError string + }{ + { + name: "name", + args: []string{"--region", "us-east-1", "--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}, + wantError: "required flag(s) \"name\" not set", + }, + { + name: "region", + args: []string{"--name", "example-cluster", "--kubeconfig", "/tmp/kubeconfig", "--output-dir", "/tmp/output"}, + wantError: "required flag(s) \"region\" not set", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + command := newRotateSigningKeyCmd(awsRotationDependencies{}) + command.SilenceErrors = true + command.SilenceUsage = true + command.SetArgs(test.args) + err := command.Execute() + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("Execute() error = %v, want error containing %q", err, test.wantError) + } + }) + } +} + +func TestAWSRotateSigningKeyCommandRejectsManualPublicationBeforeConstruction(t *testing.T) { + t.Parallel() + + command := newRotateSigningKeyCmd(awsRotationDependencies{}) + command.SilenceErrors = true + command.SilenceUsage = true + command.SetArgs([]string{ + "--name", "example-cluster", + "--region", "us-east-1", + "--kubeconfig", "/tmp/kubeconfig", + "--output-dir", "/tmp/output", + "--publication-mode", "manual", + }) + + err := command.Execute() + if err == nil || !strings.Contains(err.Error(), "supports only direct publication") { + t.Fatalf("Execute() error = %v, want direct-publication-only error", err) + } +} + +type awsRotationTestClient struct { + awsclient.Client +} + +type awsRotationTestCluster struct { + rotation.ClusterRotation +} + +type awsRotationTestTarget struct { + rotation.TargetResolver +} + +type awsRotationTestPublisher struct { + rotation.ConditionalJWKSBackend +} + +type awsRotationTestRunner struct { + ctx context.Context + options rotation.RunOptions + calls int + result rotation.RunResult +} + +func (r *awsRotationTestRunner) Run(ctx context.Context, options rotation.RunOptions) (rotation.RunResult, error) { + r.ctx = ctx + r.options = options + r.calls++ + return r.result, nil +} diff --git a/pkg/cmd/provisioning/aws/rotation_s3.go b/pkg/cmd/provisioning/aws/rotation_s3.go new file mode 100644 index 0000000000..f674cdcb67 --- /dev/null +++ b/pkg/cmd/provisioning/aws/rotation_s3.go @@ -0,0 +1,470 @@ +package aws + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/url" + "regexp" + "sort" + "strings" + "unicode" + "unicode/utf8" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + + awsclient "github.com/openshift/cloud-credential-operator/pkg/aws" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +const ( + maxS3JWKSBytes = 1 << 20 + maxS3ETagBytes = 1 << 10 + maxS3VersionIDBytes = 2 << 10 + s3RevisionPrefix = "s3-object-v1." + maxS3RevisionPayloadLen = 32 << 10 +) + +var ( + s3BucketNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]*[a-z0-9]$`) + awsRegionPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*[a-z0-9]$`) +) + +// S3TargetResolver returns one canonical identity for the configured keys.json +// object. The region is part of the identity so identically named buckets in +// different AWS partitions cannot share rotation state. +type S3TargetResolver struct { + target string +} + +// NewS3TargetResolver constructs a resolver for a ccoctl-managed S3 issuer. +func NewS3TargetResolver(region, bucket string) (*S3TargetResolver, error) { + target, err := s3TargetIdentity(region, bucket) + if err != nil { + return nil, err + } + return &S3TargetResolver{target: target}, nil +} + +// ResolveTarget implements rotation.TargetResolver. +func (r *S3TargetResolver) ResolveTarget(ctx context.Context) (string, error) { + if r == nil { + return "", fmt.Errorf("AWS S3 target resolver must not be nil") + } + if ctx == nil { + return "", fmt.Errorf("AWS S3 target resolution requires a context") + } + if err := ctx.Err(); err != nil { + return "", fmt.Errorf("resolve AWS S3 JWKS target: %w", err) + } + return r.target, nil +} + +// S3JWKSBackend reads and conditionally replaces a ccoctl-managed keys.json +// object. Its opaque revision carries the predecessor ETag plus the observed +// version ID and exact tag set used by a subsequent write. +type S3JWKSBackend struct { + client awsclient.Client + bucket string + key string + target string + ownershipTagKey string +} + +// NewS3JWKSBackend constructs the direct-publication backend for one S3 issuer. +func NewS3JWKSBackend(client awsclient.Client, region, bucket, ownerName string) (*S3JWKSBackend, error) { + if client == nil { + return nil, fmt.Errorf("AWS client must not be nil") + } + target, err := s3TargetIdentity(region, bucket) + if err != nil { + return nil, err + } + if strings.TrimSpace(ownerName) == "" || strings.TrimSpace(ownerName) != ownerName { + return nil, fmt.Errorf("AWS resource owner name must not be empty or contain surrounding whitespace") + } + if containsControl(ownerName) { + return nil, fmt.Errorf("AWS resource owner name must not contain control characters") + } + ownershipTagKey := fmt.Sprintf("%s/%s", ccoctlAWSResourceTagKeyPrefix, ownerName) + if utf8.RuneCountInString(ownershipTagKey) > 128 { + return nil, fmt.Errorf("AWS resource ownership tag key exceeds the S3 limit") + } + + return &S3JWKSBackend{ + client: client, + bucket: bucket, + key: provisioning.KeysURI, + target: target, + ownershipTagKey: ownershipTagKey, + }, nil +} + +// CheckAccess verifies that the exact object and its tags are readable and +// that the object is owned by the configured ccoctl resource. S3 has no +// mutation-free probe for PutObject authorization, so the eventual write is +// still guarded by If-Match and returns an actionable authorization failure. +// Versioned objects also require s3:GetObjectVersionTagging. +func (b *S3JWKSBackend) CheckAccess(ctx context.Context, target string) error { + if _, err := b.ReadJWKS(ctx, target); err != nil { + return fmt.Errorf("validate AWS S3 JWKS read and tagging access: %w", err) + } + return nil +} + +// ReadJWKS returns the public keys.json bytes and an opaque revision that +// records the exact object tags for publication. +func (b *S3JWKSBackend) ReadJWKS(ctx context.Context, target string) (rotation.VersionedJWKS, error) { + var versioned rotation.VersionedJWKS + if err := b.validateRequest(ctx, target); err != nil { + return versioned, err + } + + output, err := b.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: awssdk.String(b.bucket), + Key: awssdk.String(b.key), + }) + if err != nil { + return versioned, fmt.Errorf("read AWS S3 JWKS object %q (requires s3:GetObject): %w", b.target, err) + } + if output == nil || output.Body == nil { + return versioned, fmt.Errorf("read AWS S3 JWKS object %q: response body is missing", b.target) + } + + data, err := readBoundedS3Object(output.Body) + if err != nil { + return versioned, fmt.Errorf("read AWS S3 JWKS object %q body: %w", b.target, err) + } + etag := awssdk.ToString(output.ETag) + if strings.TrimSpace(etag) == "" || strings.TrimSpace(etag) != etag || containsControl(etag) { + return versioned, fmt.Errorf("read AWS S3 JWKS object %q: response ETag is missing or invalid", b.target) + } + + tagOutput, tags, err := b.readObjectTags(ctx, output.VersionId) + if err != nil { + return versioned, err + } + if !sameOptionalString(output.VersionId, tagOutput.VersionId) { + return versioned, fmt.Errorf("read AWS S3 JWKS object %q: object and tag version IDs differ", b.target) + } + if err := b.requireOwnershipTag(tags); err != nil { + return versioned, err + } + + revision, err := encodeS3Revision(s3ObjectRevision{ + ETag: etag, + VersionID: awssdk.ToString(output.VersionId), + Tags: tags, + }) + if err != nil { + return versioned, fmt.Errorf("encode AWS S3 JWKS object revision: %w", err) + } + return rotation.VersionedJWKS{Data: data, Revision: revision}, nil +} + +// PublishIfVersion replaces keys.json only when the current ETag still matches +// the predecessor revision. It performs best-effort version and tag drift +// checks immediately before the write, then copies the last-observed exact tag +// set to the new object, including the required OpenShift ownership tag. S3's +// atomic write precondition covers the object ETag, not tags or version IDs. +func (b *S3JWKSBackend) PublishIfVersion(ctx context.Context, target, revision string, data []byte) (rotation.EffectOutcome, error) { + if err := b.validateRequest(ctx, target); err != nil { + return rotation.EffectNotApplied, err + } + if len(data) == 0 { + return rotation.EffectNotApplied, fmt.Errorf("refusing to publish an empty AWS S3 JWKS object") + } + if len(data) > maxS3JWKSBytes { + return rotation.EffectNotApplied, fmt.Errorf("refusing to publish an AWS S3 JWKS object larger than the %d-byte limit", maxS3JWKSBytes) + } + + decoded, err := decodeS3Revision(revision) + if err != nil { + return rotation.EffectNotApplied, fmt.Errorf("decode AWS S3 JWKS object revision: %w", err) + } + if err := b.requireOwnershipTag(decoded.Tags); err != nil { + return rotation.EffectNotApplied, err + } + + currentTagOutput, currentTags, err := b.readObjectTags(ctx, nil) + if err != nil { + return rotation.EffectNotApplied, fmt.Errorf("recheck AWS S3 JWKS tags before publication: %w", err) + } + if awssdk.ToString(currentTagOutput.VersionId) != decoded.VersionID { + return rotation.EffectNotApplied, fmt.Errorf("AWS S3 JWKS object version changed before conditional publication") + } + if !equalS3RevisionTags(currentTags, decoded.Tags) { + return rotation.EffectNotApplied, fmt.Errorf("AWS S3 JWKS object tags changed before conditional publication") + } + + encodedTags := encodeS3ObjectTags(decoded.Tags) + _, putErr := b.client.PutObject(ctx, &s3.PutObjectInput{ + Body: bytes.NewReader(data), + Bucket: awssdk.String(b.bucket), + ContentType: awssdk.String("application/json"), + IfMatch: awssdk.String(decoded.ETag), + Key: awssdk.String(b.key), + Tagging: awssdk.String(encodedTags), + }, func(options *s3.Options) { + // A conditional replacement is not safe for transparent retry. If the + // first response is lost after S3 commits the write, a replay uses the + // stale ETag and can return a misleading precondition failure. + options.Retryer = awssdk.NopRetryer{} + }) + if putErr == nil { + return rotation.EffectSubmitted, nil + } + + err = fmt.Errorf("conditionally publish AWS S3 JWKS object %q (requires s3:PutObject and s3:PutObjectTagging): %w", b.target, putErr) + if s3WriteDefinitelyNotApplied(putErr) { + return rotation.EffectNotApplied, err + } + return rotation.EffectUnknown, err +} + +func (b *S3JWKSBackend) validateRequest(ctx context.Context, target string) error { + if b == nil || b.client == nil { + return fmt.Errorf("AWS S3 JWKS backend must not be nil") + } + if ctx == nil { + return fmt.Errorf("AWS S3 JWKS operation requires a context") + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("AWS S3 JWKS operation cancelled before submission: %w", err) + } + if target != b.target { + return fmt.Errorf("AWS S3 JWKS target does not match configured target %q", b.target) + } + return nil +} + +func (b *S3JWKSBackend) readObjectTags(ctx context.Context, versionID *string) (*s3.GetObjectTaggingOutput, []s3RevisionTag, error) { + permission := "s3:GetObjectTagging" + if awssdk.ToString(versionID) != "" { + permission = "s3:GetObjectVersionTagging" + } + output, err := b.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: awssdk.String(b.bucket), + Key: awssdk.String(b.key), + VersionId: versionID, + }) + if err != nil { + return nil, nil, fmt.Errorf("read AWS S3 JWKS object tags %q (requires %s): %w", b.target, permission, err) + } + if output == nil { + return nil, nil, fmt.Errorf("read AWS S3 JWKS object tags %q: response is missing", b.target) + } + tags, err := normalizeS3Tags(output.TagSet) + if err != nil { + return nil, nil, fmt.Errorf("read AWS S3 JWKS object tags %q: %w", b.target, err) + } + return output, tags, nil +} + +func (b *S3JWKSBackend) requireOwnershipTag(tags []s3RevisionTag) error { + for _, tag := range tags { + if tag.Key == b.ownershipTagKey && tag.Value == ownedCcoctlAWSResourceTagValue { + return nil + } + } + return fmt.Errorf("AWS S3 JWKS object %q is missing ownership tag %q=%q", b.target, b.ownershipTagKey, ownedCcoctlAWSResourceTagValue) +} + +func s3TargetIdentity(region, bucket string) (string, error) { + if err := validateAWSRegion(region); err != nil { + return "", err + } + if err := validateS3BucketName(bucket); err != nil { + return "", err + } + target := url.URL{ + Scheme: "s3", + Host: bucket, + Path: "/" + provisioning.KeysURI, + RawQuery: url.Values{"region": []string{region}}.Encode(), + } + return target.String(), nil +} + +func validateAWSRegion(region string) error { + if strings.TrimSpace(region) == "" || strings.TrimSpace(region) != region || len(region) > 63 || !awsRegionPattern.MatchString(region) { + return fmt.Errorf("AWS region %q is invalid", region) + } + return nil +} + +func validateS3BucketName(bucket string) error { + if len(bucket) < 3 || len(bucket) > 63 || !s3BucketNamePattern.MatchString(bucket) || strings.Contains(bucket, "..") || net.ParseIP(bucket) != nil { + return fmt.Errorf("AWS S3 bucket name %q is invalid", bucket) + } + return nil +} + +func readBoundedS3Object(body io.ReadCloser) ([]byte, error) { + limited := &io.LimitedReader{R: body, N: maxS3JWKSBytes + 1} + data, readErr := io.ReadAll(limited) + closeErr := body.Close() + if readErr != nil { + return nil, readErr + } + if len(data) > maxS3JWKSBytes { + return nil, fmt.Errorf("object exceeds the %d-byte JWKS limit", maxS3JWKSBytes) + } + if closeErr != nil { + return nil, closeErr + } + return data, nil +} + +type s3ObjectRevision struct { + ETag string `json:"etag"` + VersionID string `json:"versionID,omitempty"` + Tags []s3RevisionTag `json:"tags"` +} + +type s3RevisionTag struct { + Key string `json:"key"` + Value string `json:"value"` +} + +func encodeS3Revision(revision s3ObjectRevision) (string, error) { + if err := validateS3Revision(&revision); err != nil { + return "", err + } + raw, err := json.Marshal(revision) + if err != nil { + return "", err + } + payload := base64.RawURLEncoding.EncodeToString(raw) + if len(payload) > maxS3RevisionPayloadLen { + return "", fmt.Errorf("revision payload exceeds the supported size") + } + return s3RevisionPrefix + payload, nil +} + +func decodeS3Revision(encoded string) (s3ObjectRevision, error) { + var revision s3ObjectRevision + if !strings.HasPrefix(encoded, s3RevisionPrefix) { + return revision, fmt.Errorf("revision has an unsupported format") + } + payload := strings.TrimPrefix(encoded, s3RevisionPrefix) + if payload == "" || len(payload) > maxS3RevisionPayloadLen { + return revision, fmt.Errorf("revision payload is empty or too large") + } + raw, err := base64.RawURLEncoding.DecodeString(payload) + if err != nil { + return revision, fmt.Errorf("revision payload is invalid: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&revision); err != nil { + return revision, fmt.Errorf("revision payload is invalid: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return revision, fmt.Errorf("revision payload contains trailing data") + } + if err := validateS3Revision(&revision); err != nil { + return revision, err + } + return revision, nil +} + +func validateS3Revision(revision *s3ObjectRevision) error { + if revision == nil || strings.TrimSpace(revision.ETag) == "" || strings.TrimSpace(revision.ETag) != revision.ETag || len(revision.ETag) > maxS3ETagBytes || containsControl(revision.ETag) { + return fmt.Errorf("revision ETag is missing or invalid") + } + if len(revision.VersionID) > maxS3VersionIDBytes || containsControl(revision.VersionID) { + return fmt.Errorf("revision version ID is invalid") + } + if len(revision.Tags) > 10 { + return fmt.Errorf("revision contains more than 10 S3 object tags") + } + seen := make(map[string]struct{}, len(revision.Tags)) + for _, tag := range revision.Tags { + if tag.Key == "" || utf8.RuneCountInString(tag.Key) > 128 || containsControl(tag.Key) { + return fmt.Errorf("revision contains an invalid S3 object tag key") + } + if utf8.RuneCountInString(tag.Value) > 256 || containsControl(tag.Value) { + return fmt.Errorf("revision contains an invalid S3 object tag value") + } + if _, exists := seen[tag.Key]; exists { + return fmt.Errorf("revision contains duplicate S3 object tag %q", tag.Key) + } + seen[tag.Key] = struct{}{} + } + sort.Slice(revision.Tags, func(i, j int) bool { + return revision.Tags[i].Key < revision.Tags[j].Key + }) + return nil +} + +func normalizeS3Tags(tagSet []s3types.Tag) ([]s3RevisionTag, error) { + tags := make([]s3RevisionTag, 0, len(tagSet)) + for _, tag := range tagSet { + if tag.Key == nil || tag.Value == nil { + return nil, fmt.Errorf("S3 object tag has a missing key or value") + } + tags = append(tags, s3RevisionTag{Key: *tag.Key, Value: *tag.Value}) + } + revision := s3ObjectRevision{ETag: "placeholder", Tags: tags} + if err := validateS3Revision(&revision); err != nil { + return nil, err + } + return revision.Tags, nil +} + +func encodeS3ObjectTags(tags []s3RevisionTag) string { + values := make(url.Values, len(tags)) + for _, tag := range tags { + values.Set(tag.Key, tag.Value) + } + return values.Encode() +} + +func equalS3RevisionTags(left, right []s3RevisionTag) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + +func sameOptionalString(left, right *string) bool { + return awssdk.ToString(left) == awssdk.ToString(right) +} + +func containsControl(value string) bool { + for _, character := range value { + if unicode.IsControl(character) { + return true + } + } + return false +} + +func s3WriteDefinitelyNotApplied(err error) bool { + var apiError smithy.APIError + if !errors.As(err, &apiError) { + return false + } + if apiError.ErrorCode() == "PreconditionFailed" || apiError.ErrorCode() == "ConditionalRequestConflict" { + return true + } + return apiError.ErrorFault() == smithy.FaultClient +} + +var _ rotation.TargetResolver = (*S3TargetResolver)(nil) +var _ rotation.ConditionalJWKSBackend = (*S3JWKSBackend)(nil) diff --git a/pkg/cmd/provisioning/aws/rotation_s3_test.go b/pkg/cmd/provisioning/aws/rotation_s3_test.go new file mode 100644 index 0000000000..4e37b920b7 --- /dev/null +++ b/pkg/cmd/provisioning/aws/rotation_s3_test.go @@ -0,0 +1,470 @@ +package aws + +import ( + "context" + "errors" + "io" + "net/url" + "strings" + "testing" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "go.uber.org/mock/gomock" + + mockaws "github.com/openshift/cloud-credential-operator/pkg/aws/mock" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +const ( + testRotationRegion = "us-gov-west-1" + testRotationBucket = "test-cluster-oidc" + testRotationOwner = "test-cluster" +) + +func TestS3TargetResolver(t *testing.T) { + resolver, err := NewS3TargetResolver(testRotationRegion, testRotationBucket) + if err != nil { + t.Fatalf("NewS3TargetResolver() error = %v", err) + } + target, err := resolver.ResolveTarget(context.Background()) + if err != nil { + t.Fatalf("ResolveTarget() error = %v", err) + } + want := "s3://test-cluster-oidc/keys.json?region=us-gov-west-1" + if target != want { + t.Fatalf("ResolveTarget() = %q, want %q", target, want) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := resolver.ResolveTarget(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("ResolveTarget(cancelled) error = %v, want context cancellation", err) + } +} + +func TestS3TargetResolverRejectsInvalidConfiguration(t *testing.T) { + tests := []struct { + name string + region string + bucket string + }{ + {name: "empty region", bucket: testRotationBucket}, + {name: "surrounding region whitespace", region: " us-east-1", bucket: testRotationBucket}, + {name: "empty bucket", region: testRotationRegion}, + {name: "uppercase bucket", region: testRotationRegion, bucket: "Test-Cluster-OIDC"}, + {name: "IP address bucket", region: testRotationRegion, bucket: "192.0.2.1"}, + {name: "adjacent bucket periods", region: testRotationRegion, bucket: "test..bucket"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := NewS3TargetResolver(test.region, test.bucket); err == nil { + t.Fatal("NewS3TargetResolver() error = nil, want invalid configuration error") + } + }) + } +} + +func TestS3JWKSBackendReadReturnsBoundedDataAndTagBoundRevision(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + tags := []s3types.Tag{ + {Key: awssdk.String("custom.example/note"), Value: awssdk.String("keep this value")}, + {Key: awssdk.String(testS3OwnershipTagKey()), Value: awssdk.String(ownedCcoctlAWSResourceTagValue)}, + } + + client.EXPECT().GetObject(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, input *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + assertS3ObjectInput(t, input.Bucket, input.Key) + return &s3.GetObjectOutput{ + Body: io.NopCloser(strings.NewReader(`{"keys":[{"kid":"old"}]}`)), + ETag: awssdk.String(`"etag-1"`), + VersionId: awssdk.String("version-1"), + }, nil + }, + ) + client.EXPECT().GetObjectTagging(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, input *s3.GetObjectTaggingInput, _ ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + assertS3ObjectInput(t, input.Bucket, input.Key) + if got := awssdk.ToString(input.VersionId); got != "version-1" { + t.Fatalf("GetObjectTagging VersionId = %q, want version-1", got) + } + return &s3.GetObjectTaggingOutput{TagSet: tags, VersionId: awssdk.String("version-1")}, nil + }, + ) + + observed, err := backend.ReadJWKS(context.Background(), target) + if err != nil { + t.Fatalf("ReadJWKS() error = %v", err) + } + if got, want := string(observed.Data), `{"keys":[{"kid":"old"}]}`; got != want { + t.Fatalf("ReadJWKS() data = %q, want %q", got, want) + } + revision, err := decodeS3Revision(observed.Revision) + if err != nil { + t.Fatalf("decodeS3Revision() error = %v", err) + } + if revision.ETag != `"etag-1"` || revision.VersionID != "version-1" { + t.Fatalf("ReadJWKS() revision = %#v", revision) + } + wantTags := []s3RevisionTag{ + {Key: "custom.example/note", Value: "keep this value"}, + {Key: testS3OwnershipTagKey(), Value: ownedCcoctlAWSResourceTagValue}, + } + if !equalS3RevisionTags(revision.Tags, wantTags) { + t.Fatalf("ReadJWKS() revision tags = %#v, want %#v", revision.Tags, wantTags) + } +} + +func TestS3JWKSBackendReadRejectsOversizedObject(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + client.EXPECT().GetObject(gomock.Any(), gomock.Any()).Return(&s3.GetObjectOutput{ + Body: io.NopCloser(strings.NewReader(strings.Repeat("x", maxS3JWKSBytes+1))), + ETag: awssdk.String(`"etag-1"`), + }, nil) + + _, err := backend.ReadJWKS(context.Background(), target) + if err == nil || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("ReadJWKS() error = %v, want size-limit error", err) + } +} + +func TestS3JWKSBackendReadReportsVersionTaggingPermission(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + client.EXPECT().GetObject(gomock.Any(), gomock.Any()).Return(&s3.GetObjectOutput{ + Body: io.NopCloser(strings.NewReader(`{"keys":[]}`)), + ETag: awssdk.String(`"etag-1"`), + VersionId: awssdk.String("version-1"), + }, nil) + client.EXPECT().GetObjectTagging(gomock.Any(), gomock.Any()).Return(nil, &smithy.GenericAPIError{ + Code: "AccessDenied", Message: "missing version-tag permission", Fault: smithy.FaultClient, + }) + + _, err := backend.ReadJWKS(context.Background(), target) + if err == nil || !strings.Contains(err.Error(), "s3:GetObjectVersionTagging") { + t.Fatalf("ReadJWKS() error = %v, want version-tagging permission context", err) + } +} + +func TestS3JWKSBackendCheckAccessRejectsUnownedObject(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + client.EXPECT().GetObject(gomock.Any(), gomock.Any()).Return(&s3.GetObjectOutput{ + Body: io.NopCloser(strings.NewReader(`{"keys":[]}`)), + ETag: awssdk.String(`"etag-1"`), + }, nil) + client.EXPECT().GetObjectTagging(gomock.Any(), gomock.Any()).Return(&s3.GetObjectTaggingOutput{ + TagSet: []s3types.Tag{{Key: awssdk.String("Name"), Value: awssdk.String(testRotationOwner)}}, + }, nil) + + err := backend.CheckAccess(context.Background(), target) + if err == nil || !strings.Contains(err.Error(), "missing ownership tag") { + t.Fatalf("CheckAccess() error = %v, want missing ownership tag", err) + } +} + +func TestS3JWKSBackendPublishUsesETagAndPreservesExactTags(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + tags := []s3RevisionTag{ + {Key: "Name", Value: testRotationOwner}, + {Key: "custom.example/note", Value: "spaces & symbols"}, + {Key: testS3OwnershipTagKey(), Value: ownedCcoctlAWSResourceTagValue}, + } + revision := mustEncodeS3Revision(t, s3ObjectRevision{ETag: `"etag-1"`, VersionID: "version-1", Tags: tags}) + expectCurrentS3Tags(t, client, tags, "version-1") + + desired := []byte(`{"keys":[{"kid":"new"}]}`) + client.EXPECT().PutObject(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, input *s3.PutObjectInput, optionFunctions ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + assertS3ObjectInput(t, input.Bucket, input.Key) + assertS3WriteRetriesDisabled(t, optionFunctions) + if got := awssdk.ToString(input.ContentType); got != "application/json" { + t.Fatalf("PutObject ContentType = %q, want application/json", got) + } + if got := awssdk.ToString(input.IfMatch); got != `"etag-1"` { + t.Fatalf("PutObject IfMatch = %q, want quoted predecessor ETag", got) + } + body, err := io.ReadAll(input.Body) + if err != nil { + t.Fatalf("read PutObject body: %v", err) + } + if string(body) != string(desired) { + t.Fatalf("PutObject body = %q, want %q", body, desired) + } + gotTags, err := url.ParseQuery(awssdk.ToString(input.Tagging)) + if err != nil { + t.Fatalf("parse PutObject Tagging: %v", err) + } + wantTags := url.Values{ + "Name": []string{testRotationOwner}, + "custom.example/note": []string{"spaces & symbols"}, + testS3OwnershipTagKey(): []string{ownedCcoctlAWSResourceTagValue}, + } + if gotTags.Encode() != wantTags.Encode() { + t.Fatalf("PutObject tags = %q, want %q", gotTags.Encode(), wantTags.Encode()) + } + return &s3.PutObjectOutput{}, nil + }, + ) + + outcome, err := backend.PublishIfVersion(context.Background(), target, revision, desired) + if err != nil { + t.Fatalf("PublishIfVersion() error = %v", err) + } + if outcome != rotation.EffectSubmitted { + t.Fatalf("PublishIfVersion() outcome = %q, want %q", outcome, rotation.EffectSubmitted) + } +} + +func TestS3JWKSBackendPublishRejectsTagOrVersionDrift(t *testing.T) { + tests := []struct { + name string + currentVersion string + currentTags []s3RevisionTag + wantError string + }{ + { + name: "version changed", + currentVersion: "version-2", + currentTags: testS3RevisionTags(), + wantError: "version changed", + }, + { + name: "tags changed", + currentVersion: "version-1", + currentTags: append(testS3RevisionTags(), s3RevisionTag{ + Key: "new-tag", Value: "new-value", + }), + wantError: "tags changed", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + revision := mustEncodeS3Revision(t, s3ObjectRevision{ + ETag: `"etag-1"`, VersionID: "version-1", Tags: testS3RevisionTags(), + }) + expectCurrentS3Tags(t, client, test.currentTags, test.currentVersion) + + outcome, err := backend.PublishIfVersion(context.Background(), target, revision, []byte(`{"keys":[]}`)) + if outcome != rotation.EffectNotApplied { + t.Fatalf("PublishIfVersion() outcome = %q, want %q", outcome, rotation.EffectNotApplied) + } + if err == nil || !strings.Contains(err.Error(), test.wantError) { + t.Fatalf("PublishIfVersion() error = %v, want %q", err, test.wantError) + } + }) + } +} + +func TestS3JWKSBackendPublishClassifiesAWSOutcomes(t *testing.T) { + tests := []struct { + name string + putError error + wantOutcome rotation.EffectOutcome + }{ + { + name: "precondition failed", + putError: &smithy.GenericAPIError{ + Code: "PreconditionFailed", Message: "stale ETag", Fault: smithy.FaultUnknown, + }, + wantOutcome: rotation.EffectNotApplied, + }, + { + name: "conditional conflict", + putError: &smithy.GenericAPIError{ + Code: "ConditionalRequestConflict", Message: "conflicting write", Fault: smithy.FaultUnknown, + }, + wantOutcome: rotation.EffectNotApplied, + }, + { + name: "access denied", + putError: &smithy.GenericAPIError{ + Code: "AccessDenied", Message: "missing permission", Fault: smithy.FaultClient, + }, + wantOutcome: rotation.EffectNotApplied, + }, + { + name: "server outcome unknown", + putError: &smithy.GenericAPIError{ + Code: "InternalError", Message: "try again", Fault: smithy.FaultServer, + }, + wantOutcome: rotation.EffectUnknown, + }, + { + name: "transport outcome unknown", + putError: errors.New("connection closed"), + wantOutcome: rotation.EffectUnknown, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + tags := testS3RevisionTags() + revision := mustEncodeS3Revision(t, s3ObjectRevision{ + ETag: `"etag-1"`, VersionID: "version-1", Tags: tags, + }) + expectCurrentS3Tags(t, client, tags, "version-1") + client.EXPECT().PutObject(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, _ *s3.PutObjectInput, optionFunctions ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + assertS3WriteRetriesDisabled(t, optionFunctions) + return nil, test.putError + }, + ) + + outcome, err := backend.PublishIfVersion(context.Background(), target, revision, []byte(`{"keys":[]}`)) + if outcome != test.wantOutcome { + t.Fatalf("PublishIfVersion() outcome = %q, want %q", outcome, test.wantOutcome) + } + if err == nil || !strings.Contains(err.Error(), "s3:PutObject") { + t.Fatalf("PublishIfVersion() error = %v, want actionable permission context", err) + } + }) + } +} + +func TestS3JWKSBackendPublishRejectsInvalidRevisionWithoutAWSCalls(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + + outcome, err := backend.PublishIfVersion(context.Background(), target, "not-a-revision", []byte(`{"keys":[]}`)) + if outcome != rotation.EffectNotApplied { + t.Fatalf("PublishIfVersion() outcome = %q, want %q", outcome, rotation.EffectNotApplied) + } + if err == nil || !strings.Contains(err.Error(), "unsupported format") { + t.Fatalf("PublishIfVersion() error = %v, want invalid revision", err) + } +} + +func TestS3JWKSBackendPublishRejectsOversizedObjectWithoutAWSCalls(t *testing.T) { + controller := gomock.NewController(t) + client := mockaws.NewMockClient(controller) + backend, target := newTestS3JWKSBackend(t, client) + revision := mustEncodeS3Revision(t, s3ObjectRevision{ + ETag: `"etag-1"`, Tags: testS3RevisionTags(), + }) + + outcome, err := backend.PublishIfVersion(context.Background(), target, revision, make([]byte, maxS3JWKSBytes+1)) + if outcome != rotation.EffectNotApplied { + t.Fatalf("PublishIfVersion() outcome = %q, want %q", outcome, rotation.EffectNotApplied) + } + if err == nil || !strings.Contains(err.Error(), "larger than") { + t.Fatalf("PublishIfVersion() error = %v, want size-limit error", err) + } +} + +func TestS3RevisionRejectsOversizedOpaqueFields(t *testing.T) { + tests := []struct { + name string + revision s3ObjectRevision + }{ + { + name: "ETag", + revision: s3ObjectRevision{ + ETag: strings.Repeat("e", maxS3ETagBytes+1), Tags: testS3RevisionTags(), + }, + }, + { + name: "version ID", + revision: s3ObjectRevision{ + ETag: `"etag-1"`, VersionID: strings.Repeat("v", maxS3VersionIDBytes+1), Tags: testS3RevisionTags(), + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := encodeS3Revision(test.revision); err == nil { + t.Fatal("encodeS3Revision() error = nil, want bounded-field error") + } + }) + } +} + +func newTestS3JWKSBackend(t *testing.T, client *mockaws.MockClient) (*S3JWKSBackend, string) { + t.Helper() + backend, err := NewS3JWKSBackend(client, testRotationRegion, testRotationBucket, testRotationOwner) + if err != nil { + t.Fatalf("NewS3JWKSBackend() error = %v", err) + } + resolver, err := NewS3TargetResolver(testRotationRegion, testRotationBucket) + if err != nil { + t.Fatalf("NewS3TargetResolver() error = %v", err) + } + target, err := resolver.ResolveTarget(context.Background()) + if err != nil { + t.Fatalf("ResolveTarget() error = %v", err) + } + return backend, target +} + +func testS3OwnershipTagKey() string { + return ccoctlAWSResourceTagKeyPrefix + "/" + testRotationOwner +} + +func testS3RevisionTags() []s3RevisionTag { + return []s3RevisionTag{{Key: testS3OwnershipTagKey(), Value: ownedCcoctlAWSResourceTagValue}} +} + +func mustEncodeS3Revision(t *testing.T, revision s3ObjectRevision) string { + t.Helper() + encoded, err := encodeS3Revision(revision) + if err != nil { + t.Fatalf("encodeS3Revision() error = %v", err) + } + return encoded +} + +func expectCurrentS3Tags(t *testing.T, client *mockaws.MockClient, tags []s3RevisionTag, versionID string) { + t.Helper() + client.EXPECT().GetObjectTagging(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, input *s3.GetObjectTaggingInput, _ ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + assertS3ObjectInput(t, input.Bucket, input.Key) + if input.VersionId != nil { + t.Fatalf("current GetObjectTagging VersionId = %q, want no explicit version", awssdk.ToString(input.VersionId)) + } + tagSet := make([]s3types.Tag, 0, len(tags)) + for _, tag := range tags { + tagSet = append(tagSet, s3types.Tag{Key: awssdk.String(tag.Key), Value: awssdk.String(tag.Value)}) + } + return &s3.GetObjectTaggingOutput{TagSet: tagSet, VersionId: awssdk.String(versionID)}, nil + }, + ) +} + +func assertS3ObjectInput(t *testing.T, bucket, key *string) { + t.Helper() + if got := awssdk.ToString(bucket); got != testRotationBucket { + t.Fatalf("AWS S3 bucket = %q, want %q", got, testRotationBucket) + } + if got := awssdk.ToString(key); got != provisioning.KeysURI { + t.Fatalf("AWS S3 key = %q, want %q", got, provisioning.KeysURI) + } +} + +func assertS3WriteRetriesDisabled(t *testing.T, optionFunctions []func(*s3.Options)) { + t.Helper() + if len(optionFunctions) != 1 { + t.Fatalf("PutObject option functions = %d, want 1", len(optionFunctions)) + } + options := s3.Options{} + optionFunctions[0](&options) + if options.Retryer == nil || options.Retryer.MaxAttempts() != 1 { + t.Fatalf("PutObject retryer max attempts = %v, want 1", options.Retryer) + } +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/adapter.go b/pkg/cmd/provisioning/rotation/kubeadapter/adapter.go new file mode 100644 index 0000000000..922863ab93 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/adapter.go @@ -0,0 +1,239 @@ +package kubeadapter + +import ( + "context" + "fmt" + "strings" + "time" + + configv1 "github.com/openshift/api/config/v1" + configclient "github.com/openshift/client-go/config/clientset/versioned" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +const ( + cloudCredentialOperatorNamespace = "openshift-cloud-credential-operator" + kubeAPIServerOperatorNamespace = "openshift-kube-apiserver-operator" + kubeAPIServerNamespace = "openshift-kube-apiserver" + + nextSignerSecretName = "next-bound-service-account-signing-key" + publicSignerCMName = "bound-sa-token-signing-certs" + + defaultPollInterval = 10 * time.Second + defaultPreflightStablePeriod = 5 * time.Second + defaultStablePeriod = 5 * time.Minute +) + +var ( + kubeAPIServerGVR = schema.GroupVersionResource{ + Group: "operator.openshift.io", Version: "v1", Resource: "kubeapiservers", + } + machineConfigPoolGVR = schema.GroupVersionResource{ + Group: "machineconfiguration.openshift.io", Version: "v1", Resource: "machineconfigpools", + } + machineConfigGVR = schema.GroupVersionResource{ + Group: "machineconfiguration.openshift.io", Version: "v1", Resource: "machineconfigs", + } +) + +// Options controls polling and continuous-stability windows. New uses the +// production defaults; NewWithOptions exists so callers can select a stricter +// policy and unit tests can avoid wall-clock waits. +type Options struct { + PollInterval time.Duration + PreflightStablePeriod time.Duration + StablePeriod time.Duration +} + +// DefaultOptions returns the stability windows used by the documented manual +// rotation procedure. +func DefaultOptions() Options { + return Options{ + PollInterval: defaultPollInterval, + PreflightStablePeriod: defaultPreflightStablePeriod, + StablePeriod: defaultStablePeriod, + } +} + +type clusterVersionGetter interface { + Get(context.Context, string, metav1.GetOptions) (*configv1.ClusterVersion, error) +} + +type clusterOperatorLister interface { + List(context.Context, metav1.ListOptions) (*configv1.ClusterOperatorList, error) +} + +type secretMetadataGetter interface { + Get(context.Context, string, string) (*metav1.PartialObjectMetadata, error) +} + +type resourceClient interface { + Get(context.Context, schema.GroupVersionResource, string, metav1.GetOptions) (*unstructured.Unstructured, error) + List(context.Context, schema.GroupVersionResource, metav1.ListOptions) (*unstructured.UnstructuredList, error) + Create(context.Context, schema.GroupVersionResource, *unstructured.Unstructured, metav1.CreateOptions) (*unstructured.Unstructured, error) + Update(context.Context, schema.GroupVersionResource, *unstructured.Unstructured, metav1.UpdateOptions) (*unstructured.Unstructured, error) +} + +type dynamicResourceClient struct { + client dynamic.Interface +} + +func (c dynamicResourceClient) Get(ctx context.Context, resource schema.GroupVersionResource, name string, options metav1.GetOptions) (*unstructured.Unstructured, error) { + return c.client.Resource(resource).Get(ctx, name, options) +} + +func (c dynamicResourceClient) List(ctx context.Context, resource schema.GroupVersionResource, options metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return c.client.Resource(resource).List(ctx, options) +} + +func (c dynamicResourceClient) Create(ctx context.Context, resource schema.GroupVersionResource, object *unstructured.Unstructured, options metav1.CreateOptions) (*unstructured.Unstructured, error) { + return c.client.Resource(resource).Create(ctx, object, options) +} + +func (c dynamicResourceClient) Update(ctx context.Context, resource schema.GroupVersionResource, object *unstructured.Unstructured, options metav1.UpdateOptions) (*unstructured.Unstructured, error) { + return c.client.Resource(resource).Update(ctx, object, options) +} + +// Adapter implements the provider-independent cluster half of signer-key +// rotation. It never reads a signer Secret payload. +type Adapter struct { + kube kubernetes.Interface + clusterVersions clusterVersionGetter + clusterOperators clusterOperatorLister + resources resourceClient + secretMetadata secretMetadataGetter + options Options + now func() time.Time +} + +var _ rotation.ClusterRotation = (*Adapter)(nil) + +// New constructs an adapter from one explicit kubeconfig path. +func New(kubeconfig string) (*Adapter, error) { + return NewWithOptions(kubeconfig, DefaultOptions()) +} + +// NewWithOptions constructs an adapter with explicit polling and stability +// settings. An empty kubeconfig is rejected; rotation never falls back to an +// ambient or in-cluster identity. +func NewWithOptions(kubeconfig string, options Options) (*Adapter, error) { + if strings.TrimSpace(kubeconfig) == "" { + return nil, fmt.Errorf("rotation kubeconfig path must not be empty") + } + if options.PollInterval <= 0 { + return nil, fmt.Errorf("rotation poll interval must be greater than zero") + } + if options.PreflightStablePeriod < 0 || options.StablePeriod < 0 { + return nil, fmt.Errorf("rotation stability periods must not be negative") + } + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, fmt.Errorf("load rotation kubeconfig: %w", err) + } + config = rest.CopyConfig(config) + config.UserAgent = "ccoctl-signer-key-rotation" + + httpClient, err := rest.HTTPClientFor(config) + if err != nil { + return nil, fmt.Errorf("build rotation Kubernetes transport: %w", err) + } + kube, err := kubernetes.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, fmt.Errorf("build rotation Kubernetes client: %w", err) + } + configClient, err := configclient.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, fmt.Errorf("build rotation OpenShift config client: %w", err) + } + dynamicClient, err := dynamic.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, fmt.Errorf("build rotation dynamic client: %w", err) + } + + return newAdapter( + kube, + configClient.ConfigV1().ClusterVersions(), + configClient.ConfigV1().ClusterOperators(), + dynamicResourceClient{client: dynamicClient}, + &strictSecretMetadataClient{restClient: kube.CoreV1().RESTClient()}, + options, + ), nil +} + +func newAdapter(kube kubernetes.Interface, versions clusterVersionGetter, operators clusterOperatorLister, resources resourceClient, metadata secretMetadataGetter, options Options) *Adapter { + return &Adapter{ + kube: kube, + clusterVersions: versions, + clusterOperators: operators, + resources: resources, + secretMetadata: metadata, + options: options, + now: time.Now, + } +} + +func (a *Adapter) waitUntil(ctx context.Context, condition func(context.Context) (bool, error)) error { + for { + done, err := condition(ctx) + if err != nil { + return err + } + if done { + return nil + } + + timer := time.NewTimer(a.options.PollInterval) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return ctx.Err() + case <-timer.C: + } + } +} + +func (a *Adapter) waitForContinuousStability(ctx context.Context, period time.Duration, check func(context.Context) (bool, error)) error { + var stableSince time.Time + return a.waitUntil(ctx, func(ctx context.Context) (bool, error) { + stable, err := check(ctx) + if err != nil { + return false, err + } + if !stable { + stableSince = time.Time{} + return false, nil + } + if period == 0 { + return true, nil + } + if stableSince.IsZero() { + stableSince = a.now() + return false, nil + } + return a.now().Sub(stableSince) >= period, nil + }) +} + +func mutationOutcome(err error) rotation.EffectOutcome { + if err == nil { + return rotation.EffectSubmitted + } + if isDefinitiveMutationRejection(err) { + return rotation.EffectNotApplied + } + return rotation.EffectUnknown +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/guard.go b/pkg/cmd/provisioning/rotation/kubeadapter/guard.go new file mode 100644 index 0000000000..eaf8cd5fe2 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/guard.go @@ -0,0 +1,264 @@ +package kubeadapter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +const ( + guardConfigMapName = "cco-signer-key-rotation-guard" + guardStateDataKey = "state.json" + guardSchemaVersion = 1 + + managedByLabelKey = "app.kubernetes.io/managed-by" + managedByLabelValue = "cloud-credential-operator" +) + +type guardState struct { + SchemaVersion int `json:"schemaVersion"` + ScopeID string `json:"scopeID"` + ActiveOperationID string `json:"activeOperationID,omitempty"` + CompletedOperationIDs []string `json:"completedOperationIDs,omitempty"` +} + +func (a *Adapter) ObserveRotationGuard(ctx context.Context, reference rotation.RotationGuardReference) (rotation.RotationGuardObservation, error) { + if err := validateGuardReference(reference); err != nil { + return rotation.RotationGuardObservation{}, err + } + configMap, err := a.kube.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace).Get(ctx, guardConfigMapName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return rotation.RotationGuardObservation{Status: rotation.RotationGuardNotFound}, nil + } + if err != nil { + return rotation.RotationGuardObservation{}, fmt.Errorf("read signer-rotation guard ConfigMap: %w", err) + } + state, err := decodeGuardState(configMap, reference.ScopeID) + if err != nil { + return rotation.RotationGuardObservation{}, err + } + if slices.Contains(state.CompletedOperationIDs, reference.OperationID) { + return rotation.RotationGuardObservation{Status: rotation.RotationGuardCompleted, OperationID: reference.OperationID}, nil + } + switch state.ActiveOperationID { + case "": + return rotation.RotationGuardObservation{Status: rotation.RotationGuardNotFound}, nil + case reference.OperationID: + return rotation.RotationGuardObservation{Status: rotation.RotationGuardHeld, OperationID: reference.OperationID}, nil + default: + return rotation.RotationGuardObservation{Status: rotation.RotationGuardOwnedByOther, OperationID: state.ActiveOperationID}, nil + } +} + +func (a *Adapter) AcquireRotationGuard(ctx context.Context, reference rotation.RotationGuardReference) (rotation.EffectOutcome, error) { + if err := validateGuardReference(reference); err != nil { + return rotation.EffectNotApplied, err + } + client := a.kube.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace) + configMap, err := client.Get(ctx, guardConfigMapName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + state := guardState{SchemaVersion: guardSchemaVersion, ScopeID: reference.ScopeID, ActiveOperationID: reference.OperationID} + created := newGuardConfigMap(state) + _, createErr := client.Create(ctx, created, metav1.CreateOptions{}) + return mutationOutcome(createErr), createErr + } + if err != nil { + return rotation.EffectNotApplied, fmt.Errorf("read signer-rotation guard before acquisition: %w", err) + } + state, err := decodeGuardState(configMap, reference.ScopeID) + if err != nil { + return rotation.EffectNotApplied, err + } + if slices.Contains(state.CompletedOperationIDs, reference.OperationID) { + return rotation.EffectNotApplied, fmt.Errorf("signer-rotation operation %q is already completed", reference.OperationID) + } + switch state.ActiveOperationID { + case reference.OperationID: + return rotation.EffectSubmitted, nil + case "": + state.ActiveOperationID = reference.OperationID + default: + return rotation.EffectNotApplied, fmt.Errorf("signer-rotation guard is owned by operation %q", state.ActiveOperationID) + } + updated, err := guardConfigMapWithState(configMap, state) + if err != nil { + return rotation.EffectNotApplied, err + } + _, updateErr := client.Update(ctx, updated, metav1.UpdateOptions{}) + return mutationOutcome(updateErr), updateErr +} + +func (a *Adapter) ReleaseRotationGuard(ctx context.Context, reference rotation.RotationGuardReference) (rotation.EffectOutcome, error) { + if err := validateGuardReference(reference); err != nil { + return rotation.EffectNotApplied, err + } + client := a.kube.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace) + configMap, err := client.Get(ctx, guardConfigMapName, metav1.GetOptions{}) + if err != nil { + return rotation.EffectNotApplied, fmt.Errorf("read signer-rotation guard before release: %w", err) + } + state, err := decodeGuardState(configMap, reference.ScopeID) + if err != nil { + return rotation.EffectNotApplied, err + } + if slices.Contains(state.CompletedOperationIDs, reference.OperationID) { + return rotation.EffectSubmitted, nil + } + if state.ActiveOperationID != reference.OperationID { + if state.ActiveOperationID == "" { + return rotation.EffectNotApplied, fmt.Errorf("signer-rotation guard is not held") + } + return rotation.EffectNotApplied, fmt.Errorf("signer-rotation guard is owned by operation %q", state.ActiveOperationID) + } + + state.ActiveOperationID = "" + state.CompletedOperationIDs = append(state.CompletedOperationIDs, reference.OperationID) + slices.Sort(state.CompletedOperationIDs) + updated, err := guardConfigMapWithState(configMap, state) + if err != nil { + return rotation.EffectNotApplied, err + } + _, updateErr := client.Update(ctx, updated, metav1.UpdateOptions{}) + return mutationOutcome(updateErr), updateErr +} + +func (a *Adapter) requireGuardHeld(ctx context.Context, reference rotation.RotationGuardReference) error { + observation, err := a.ObserveRotationGuard(ctx, reference) + if err != nil { + return err + } + if observation.Status != rotation.RotationGuardHeld { + return fmt.Errorf("signer-rotation guard operation %q is not held: observed %q", reference.OperationID, observation.Status) + } + return nil +} + +func newGuardConfigMap(state guardState) *corev1.ConfigMap { + encoded, err := json.Marshal(state) + if err != nil { + panic(err) + } + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cloudCredentialOperatorNamespace, + Name: guardConfigMapName, + Labels: map[string]string{managedByLabelKey: managedByLabelValue}, + }, + Data: map[string]string{guardStateDataKey: string(encoded)}, + } +} + +func guardConfigMapWithState(current *corev1.ConfigMap, state guardState) (*corev1.ConfigMap, error) { + encoded, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("encode signer-rotation guard state: %w", err) + } + updated := current.DeepCopy() + updated.Data = map[string]string{guardStateDataKey: string(encoded)} + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + updated.Labels[managedByLabelKey] = managedByLabelValue + return updated, nil +} + +func decodeGuardState(configMap *corev1.ConfigMap, expectedScopeID string) (guardState, error) { + var state guardState + if configMap == nil { + return state, fmt.Errorf("signer-rotation guard ConfigMap is nil") + } + if configMap.Namespace != cloudCredentialOperatorNamespace || configMap.Name != guardConfigMapName { + return state, fmt.Errorf("unexpected signer-rotation guard ConfigMap %q/%q", configMap.Namespace, configMap.Name) + } + if configMap.DeletionTimestamp != nil { + return state, fmt.Errorf("signer-rotation guard ConfigMap is being deleted") + } + if len(configMap.BinaryData) != 0 { + return state, fmt.Errorf("signer-rotation guard ConfigMap contains unsupported binary data") + } + if len(configMap.Data) != 1 { + return state, fmt.Errorf("signer-rotation guard ConfigMap must contain only %q", guardStateDataKey) + } + raw, exists := configMap.Data[guardStateDataKey] + if !exists { + return state, fmt.Errorf("signer-rotation guard ConfigMap is missing %q", guardStateDataKey) + } + decoder := json.NewDecoder(bytes.NewBufferString(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&state); err != nil { + return state, fmt.Errorf("decode signer-rotation guard state: %w", err) + } + if err := ensureJSONEOF(decoder); err != nil { + return state, fmt.Errorf("decode signer-rotation guard state: %w", err) + } + if state.SchemaVersion != guardSchemaVersion { + return state, fmt.Errorf("unsupported signer-rotation guard schema version %d", state.SchemaVersion) + } + if err := validateSHA256(state.ScopeID); err != nil { + return state, fmt.Errorf("invalid signer-rotation guard scope: %w", err) + } + if state.ScopeID != expectedScopeID { + return state, fmt.Errorf("signer-rotation guard scope %q does not match expected scope %q", state.ScopeID, expectedScopeID) + } + if state.ActiveOperationID != "" { + if err := validateSHA256(state.ActiveOperationID); err != nil { + return state, fmt.Errorf("invalid active signer-rotation operation: %w", err) + } + } + completed := make(map[string]struct{}, len(state.CompletedOperationIDs)) + for _, operationID := range state.CompletedOperationIDs { + if err := validateSHA256(operationID); err != nil { + return state, fmt.Errorf("invalid completed signer-rotation operation: %w", err) + } + if _, duplicate := completed[operationID]; duplicate { + return state, fmt.Errorf("duplicate completed signer-rotation operation %q", operationID) + } + completed[operationID] = struct{}{} + } + if _, completedWhileActive := completed[state.ActiveOperationID]; state.ActiveOperationID != "" && completedWhileActive { + return state, fmt.Errorf("active signer-rotation operation %q is also completed", state.ActiveOperationID) + } + return state, nil +} + +func validateGuardReference(reference rotation.RotationGuardReference) error { + if err := validateSHA256(reference.ScopeID); err != nil { + return fmt.Errorf("invalid signer-rotation guard scope: %w", err) + } + if err := validateSHA256(reference.OperationID); err != nil { + return fmt.Errorf("invalid signer-rotation guard operation: %w", err) + } + return nil +} + +func validateSHA256(value string) error { + if len(value) != 64 { + return fmt.Errorf("must be 64 lowercase hexadecimal characters") + } + for _, character := range value { + if !strings.ContainsRune("0123456789abcdef", character) { + return fmt.Errorf("must be 64 lowercase hexadecimal characters") + } + } + return nil +} + +func isDefinitiveMutationRejection(err error) bool { + return apierrors.IsAlreadyExists(err) || + apierrors.IsConflict(err) || + apierrors.IsInvalid(err) || + apierrors.IsBadRequest(err) || + apierrors.IsForbidden(err) || + apierrors.IsUnauthorized(err) || + apierrors.IsNotFound(err) || + apierrors.IsMethodNotSupported(err) +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/metadata.go b/pkg/cmd/provisioning/rotation/kubeadapter/metadata.go new file mode 100644 index 0000000000..4161ea9d10 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/metadata.go @@ -0,0 +1,75 @@ +package kubeadapter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" +) + +const partialObjectMetadataAccept = "application/json;as=PartialObjectMetadata;g=meta.k8s.io;v=v1" + +type strictSecretMetadataClient struct { + restClient rest.Interface +} + +// Get intentionally does not use client-go's metadata client. That client +// advertises plain application/json as a fallback, which permits a server to +// return the full Secret. This request accepts only PartialObjectMetadata and +// validates the returned envelope before exposing metadata to the adapter. +func (c *strictSecretMetadataClient) Get(ctx context.Context, namespace, name string) (*metav1.PartialObjectMetadata, error) { + if c == nil || c.restClient == nil { + return nil, fmt.Errorf("strict Secret metadata client is not configured") + } + result := c.restClient.Get(). + Namespace(namespace). + Resource("secrets"). + Name(name). + SetHeader("Accept", partialObjectMetadataAccept). + Do(ctx) + + var contentType string + result.ContentType(&contentType) + raw, err := result.Raw() + if err != nil { + return nil, err + } + mediaType, _, err := mime.ParseMediaType(contentType) + if err != nil || mediaType != "application/json" { + return nil, fmt.Errorf("Secret metadata request returned unsupported content type %q", contentType) + } + + var metadata metav1.PartialObjectMetadata + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&metadata); err != nil { + return nil, fmt.Errorf("decode strict PartialObjectMetadata response: %w", err) + } + if err := ensureJSONEOF(decoder); err != nil { + return nil, fmt.Errorf("decode strict PartialObjectMetadata response: %w", err) + } + if metadata.APIVersion != metav1.SchemeGroupVersion.String() || metadata.Kind != "PartialObjectMetadata" { + return nil, fmt.Errorf("Secret metadata request returned %s %q instead of meta.k8s.io/v1 PartialObjectMetadata", metadata.APIVersion, metadata.Kind) + } + if metadata.Name != name || metadata.Namespace != namespace { + return nil, fmt.Errorf("Secret metadata response identified %q/%q instead of %q/%q", metadata.Namespace, metadata.Name, namespace, name) + } + return metadata.DeepCopy(), nil +} + +func ensureJSONEOF(decoder *json.Decoder) error { + var trailing any + err := decoder.Decode(&trailing) + if err == io.EOF { + return nil + } + if err == nil { + return fmt.Errorf("response contains trailing JSON values") + } + return err +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/metadata_guard_signer_test.go b/pkg/cmd/provisioning/rotation/kubeadapter/metadata_guard_signer_test.go new file mode 100644 index 0000000000..ef97413284 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/metadata_guard_signer_test.go @@ -0,0 +1,227 @@ +package kubeadapter + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + k8stesting "k8s.io/client-go/testing" + + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +func TestStrictSecretMetadataClientRequestsOnlyPartialObjectMetadata(t *testing.T) { + wantUID := types.UID("8bd50c04-9718-4ef0-8af5-9181e7db48b2") + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/api/v1/namespaces/openshift-kube-apiserver-operator/secrets/next-bound-service-account-signing-key" { + t.Errorf("request path = %q", request.URL.Path) + } + if got := request.Header.Get("Accept"); got != partialObjectMetadataAccept { + t.Errorf("Accept header = %q, want %q", got, partialObjectMetadataAccept) + } + response.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(response).Encode(&metav1.PartialObjectMetadata{ + TypeMeta: metav1.TypeMeta{APIVersion: "meta.k8s.io/v1", Kind: "PartialObjectMetadata"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: kubeAPIServerOperatorNamespace, + Name: nextSignerSecretName, + UID: wantUID, + ResourceVersion: "17", + }, + }) + })) + defer server.Close() + + client := strictMetadataClientForServer(t, server) + metadata, err := client.Get(context.Background(), kubeAPIServerOperatorNamespace, nextSignerSecretName) + if err != nil { + t.Fatalf("Get() returned unexpected error: %v", err) + } + if metadata.UID != wantUID || metadata.ResourceVersion != "17" { + t.Fatalf("Get() = %#v", metadata) + } +} + +func TestStrictSecretMetadataClientRejectsFullSecretFallback(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(response).Encode(&corev1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Namespace: kubeAPIServerOperatorNamespace, Name: nextSignerSecretName}, + Data: map[string][]byte{"private-key": []byte("must-not-be-accepted")}, + }) + })) + defer server.Close() + + _, err := strictMetadataClientForServer(t, server).Get(context.Background(), kubeAPIServerOperatorNamespace, nextSignerSecretName) + if err == nil || (!strings.Contains(err.Error(), "unknown field") && !strings.Contains(err.Error(), "instead of")) { + t.Fatalf("Get() error = %v, want strict full-object rejection", err) + } +} + +func strictMetadataClientForServer(t *testing.T, server *httptest.Server) *strictSecretMetadataClient { + t.Helper() + clientset, err := kubernetes.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatalf("build Kubernetes client: %v", err) + } + return &strictSecretMetadataClient{restClient: clientset.CoreV1().RESTClient()} +} + +func TestRotationGuardPreservesCompletionHistoryAcrossLaterOwnership(t *testing.T) { + ctx := context.Background() + client := fake.NewSimpleClientset() + adapter := &Adapter{kube: client} + first := rotation.RotationGuardReference{ScopeID: strings.Repeat("a", 64), OperationID: strings.Repeat("b", 64)} + second := rotation.RotationGuardReference{ScopeID: first.ScopeID, OperationID: strings.Repeat("c", 64)} + + if outcome, err := adapter.AcquireRotationGuard(ctx, first); err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("AcquireRotationGuard(first) = %q, %v", outcome, err) + } + if observation, err := adapter.ObserveRotationGuard(ctx, first); err != nil || observation.Status != rotation.RotationGuardHeld { + t.Fatalf("ObserveRotationGuard(first held) = %#v, %v", observation, err) + } + if outcome, err := adapter.ReleaseRotationGuard(ctx, first); err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("ReleaseRotationGuard(first) = %q, %v", outcome, err) + } + if outcome, err := adapter.AcquireRotationGuard(ctx, second); err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("AcquireRotationGuard(second) = %q, %v", outcome, err) + } + firstObservation, err := adapter.ObserveRotationGuard(ctx, first) + if err != nil || firstObservation.Status != rotation.RotationGuardCompleted { + t.Fatalf("ObserveRotationGuard(first completed) = %#v, %v", firstObservation, err) + } + secondObservation, err := adapter.ObserveRotationGuard(ctx, second) + if err != nil || secondObservation.Status != rotation.RotationGuardHeld { + t.Fatalf("ObserveRotationGuard(second held) = %#v, %v", secondObservation, err) + } +} + +func TestAcquireRotationGuardUsesResourceVersionCAS(t *testing.T) { + ctx := context.Background() + guard := rotation.RotationGuardReference{ScopeID: strings.Repeat("a", 64), OperationID: strings.Repeat("b", 64)} + configMap := newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID}) + configMap.ResourceVersion = "17" + client := fake.NewSimpleClientset(configMap) + var submittedResourceVersion string + client.Fake.PrependReactor("update", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + submitted := action.(k8stesting.UpdateAction).GetObject().(*corev1.ConfigMap) + submittedResourceVersion = submitted.ResourceVersion + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "configmaps"}, guardConfigMapName, errors.New("concurrent owner")) + }) + adapter := &Adapter{kube: client} + + outcome, err := adapter.AcquireRotationGuard(ctx, guard) + if outcome != rotation.EffectNotApplied || !apierrors.IsConflict(err) { + t.Fatalf("AcquireRotationGuard() = %q, %v; want definitive conflict", outcome, err) + } + if submittedResourceVersion != "17" { + t.Fatalf("submitted resourceVersion = %q, want 17", submittedResourceVersion) + } + observed, err := adapter.ObserveRotationGuard(ctx, guard) + if err != nil || observed.Status != rotation.RotationGuardNotFound { + t.Fatalf("guard after rejected CAS = %#v, %v", observed, err) + } +} + +func TestRequestReplacementUsesUIDAndResourceVersionPreconditions(t *testing.T) { + guard := rotation.RotationGuardReference{ScopeID: strings.Repeat("a", 64), OperationID: strings.Repeat("b", 64)} + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: kubeAPIServerOperatorNamespace, Name: nextSignerSecretName}}, + ) + var deleteOptions metav1.DeleteOptions + client.Fake.PrependReactor("delete", "secrets", func(action k8stesting.Action) (bool, runtime.Object, error) { + deleteOptions = action.(k8stesting.DeleteAction).GetDeleteOptions() + return false, nil, nil + }) + adapter := &Adapter{kube: client} + reference := rotation.SignerObjectReference{UID: "old-signer-uid", ResourceVersion: "31"} + + outcome, err := adapter.RequestReplacement(context.Background(), guard, reference) + if err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("RequestReplacement() = %q, %v", outcome, err) + } + if deleteOptions.Preconditions == nil || deleteOptions.Preconditions.UID == nil || deleteOptions.Preconditions.ResourceVersion == nil { + t.Fatalf("delete preconditions = %#v", deleteOptions.Preconditions) + } + if string(*deleteOptions.Preconditions.UID) != reference.UID || *deleteOptions.Preconditions.ResourceVersion != reference.ResourceVersion { + t.Fatalf("delete preconditions = %#v, want UID %q and RV %q", deleteOptions.Preconditions, reference.UID, reference.ResourceVersion) + } +} + +func TestRequestReplacementClassifiesDefinitiveAndAmbiguousFailures(t *testing.T) { + guard := rotation.RotationGuardReference{ScopeID: strings.Repeat("a", 64), OperationID: strings.Repeat("b", 64)} + reference := rotation.SignerObjectReference{UID: "old-signer-uid", ResourceVersion: "31"} + for _, test := range []struct { + name string + deleteError error + wantOutcome rotation.EffectOutcome + }{ + { + name: "precondition conflict", + deleteError: apierrors.NewConflict(schema.GroupResource{Resource: "secrets"}, nextSignerSecretName, errors.New("UID changed")), + wantOutcome: rotation.EffectNotApplied, + }, + { + name: "connection lost", + deleteError: errors.New("connection lost after request submission"), + wantOutcome: rotation.EffectUnknown, + }, + } { + t.Run(test.name, func(t *testing.T) { + client := fake.NewSimpleClientset(newGuardConfigMap(guardState{ + SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID, + })) + client.Fake.PrependReactor("delete", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, test.deleteError + }) + adapter := &Adapter{kube: client} + outcome, err := adapter.RequestReplacement(context.Background(), guard, reference) + if outcome != test.wantOutcome || !errors.Is(err, test.deleteError) { + t.Fatalf("RequestReplacement() = %q, %v", outcome, err) + } + }) + } +} + +func TestObservePublicSignerBundleReturnsEveryNamedEntry(t *testing.T) { + client := fake.NewSimpleClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kubeAPIServerNamespace, + Name: publicSignerCMName, + UID: types.UID("public-config-map-uid"), + ResourceVersion: "22", + }, + Data: map[string]string{ + "service-account-002.pub": "second", + "service-account-001.pub": "first", + }, + }) + adapter := &Adapter{kube: client} + + observed, err := adapter.ObservePublicSignerBundle(context.Background(), nil) + if err != nil { + t.Fatalf("ObservePublicSignerBundle() returned unexpected error: %v", err) + } + if observed.ConfigMapUID != "public-config-map-uid" || observed.ConfigMapResourceVersion != "22" || len(observed.Signers) != 2 { + t.Fatalf("ObservePublicSignerBundle() = %#v", observed) + } + if observed.Signers[0].Name != "service-account-001.pub" || string(observed.Signers[0].PublicKeyPEM) != "first" || + observed.Signers[1].Name != "service-account-002.pub" || string(observed.Signers[1].PublicKeyPEM) != "second" { + t.Fatalf("signer entries = %#v", observed.Signers) + } +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/preflight.go b/pkg/cmd/provisioning/rotation/kubeadapter/preflight.go new file mode 100644 index 0000000000..a355209c62 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/preflight.go @@ -0,0 +1,365 @@ +package kubeadapter + +import ( + "context" + "fmt" + "reflect" + "strings" + + configv1 "github.com/openshift/api/config/v1" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +type requiredAccess struct { + verb string + group string + resource string + namespace string + name string +} + +var preflightAccess = []requiredAccess{ + {verb: "get", group: "config.openshift.io", resource: "clusterversions", name: "version"}, + {verb: "list", group: "config.openshift.io", resource: "clusteroperators"}, + {verb: "get", group: "", resource: "namespaces", name: cloudCredentialOperatorNamespace}, + {verb: "get", group: "", resource: "secrets", namespace: kubeAPIServerOperatorNamespace, name: nextSignerSecretName}, + {verb: "delete", group: "", resource: "secrets", namespace: kubeAPIServerOperatorNamespace, name: nextSignerSecretName}, + {verb: "get", group: "", resource: "configmaps", namespace: kubeAPIServerNamespace, name: publicSignerCMName}, + // Revision ConfigMap names are discovered only after the kube-apiserver + // operator rolls out a new revision, so this access cannot be name-scoped. + {verb: "get", group: "", resource: "configmaps", namespace: kubeAPIServerNamespace}, + {verb: "get", group: "", resource: "configmaps", namespace: cloudCredentialOperatorNamespace}, + {verb: "create", group: "", resource: "configmaps", namespace: cloudCredentialOperatorNamespace}, + {verb: "update", group: "", resource: "configmaps", namespace: cloudCredentialOperatorNamespace}, + {verb: "get", group: "", resource: "nodes"}, + {verb: "list", group: "", resource: "nodes"}, + {verb: "get", group: "operator.openshift.io", resource: "kubeapiservers", name: "cluster"}, + {verb: "get", group: "machineconfiguration.openshift.io", resource: "machineconfigpools"}, + {verb: "list", group: "machineconfiguration.openshift.io", resource: "machineconfigpools"}, + {verb: "get", group: "machineconfiguration.openshift.io", resource: "machineconfigs"}, + {verb: "create", group: "machineconfiguration.openshift.io", resource: "machineconfigs"}, + {verb: "update", group: "machineconfiguration.openshift.io", resource: "machineconfigs"}, +} + +func (a *Adapter) Preflight(ctx context.Context) (rotation.ClusterPreflight, error) { + if err := a.validateClients(); err != nil { + return rotation.ClusterPreflight{}, err + } + for _, access := range preflightAccess { + if err := a.requireAccess(ctx, access); err != nil { + return rotation.ClusterPreflight{}, err + } + } + if _, err := a.kube.CoreV1().Namespaces().Get(ctx, cloudCredentialOperatorNamespace, metav1.GetOptions{}); err != nil { + return rotation.ClusterPreflight{}, fmt.Errorf("read rotation guard namespace: %w", err) + } + + clusterIdentity, err := a.clusterIdentity(ctx) + if err != nil { + return rotation.ClusterPreflight{}, err + } + metadata, err := a.secretMetadata.Get(ctx, kubeAPIServerOperatorNamespace, nextSignerSecretName) + if err != nil { + return rotation.ClusterPreflight{}, fmt.Errorf("verify strict signer Secret metadata access: %w", err) + } + if metadata.UID == "" || metadata.ResourceVersion == "" { + return rotation.ClusterPreflight{}, fmt.Errorf("next signer Secret metadata is missing UID or resourceVersion") + } + if _, err := a.ObservePublicSignerBundle(ctx, nil); err != nil { + return rotation.ClusterPreflight{}, fmt.Errorf("verify public signer ConfigMap access: %w", err) + } + if _, err := a.resources.Get(ctx, kubeAPIServerGVR, "cluster", metav1.GetOptions{}); err != nil { + return rotation.ClusterPreflight{}, fmt.Errorf("read kube-apiserver operator status: %w", err) + } + + err = a.waitForContinuousStability(ctx, a.options.PreflightStablePeriod, func(ctx context.Context) (bool, error) { + operatorsStable, err := a.clusterOperatorsStable(ctx) + if err != nil || !operatorsStable { + return false, err + } + return a.allMachineConfigPoolsStable(ctx) + }) + if err != nil { + return rotation.ClusterPreflight{}, fmt.Errorf("wait for stable cluster preflight: %w", err) + } + if _, err := a.prepareRebootPlan(ctx); err != nil { + return rotation.ClusterPreflight{}, fmt.Errorf("validate node reboot topology during preflight: %w", err) + } + return rotation.ClusterPreflight{ClusterIdentity: clusterIdentity}, nil +} + +func (a *Adapter) Reconcile(ctx context.Context, guard rotation.RotationGuardReference, expectation rotation.ClusterExpectation) error { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return err + } + identity, err := a.clusterIdentity(ctx) + if err != nil { + return err + } + if identity != expectation.ClusterIdentity { + return fmt.Errorf("cluster identity %q does not match checkpoint identity %q", identity, expectation.ClusterIdentity) + } + if expectation.RotationGuard != guard { + return fmt.Errorf("cluster expectation contains a different signer-rotation guard") + } + if expectation.Phase == rotation.PhaseSignerRolloutStable { + if expectation.ReplacementSigner == nil { + return fmt.Errorf("signer-rollout checkpoint is missing replacement signer evidence") + } + stable, err := a.signerRolloutStable(ctx, expectation.ReplacementSigner.Entry.KeyID) + if err != nil { + return fmt.Errorf("reconcile signer rollout stability: %w", err) + } + if !stable { + return fmt.Errorf("checkpoint records signer rollout stability but the cluster is not currently stable") + } + } + + if expectation.RebootIntent != nil { + observation, err := a.ObserveReboot(ctx, guard, expectation.RebootIntent.ID) + if err != nil { + return err + } + if observation.CanonicalIntent != nil && !reflect.DeepEqual(*observation.CanonicalIntent, *expectation.RebootIntent) { + return fmt.Errorf("cluster-canonical reboot intent differs from checkpoint intent") + } + if phaseAtLeast(expectation.Phase, rotation.PhaseNodesRebooted) && observation.Status != rotation.RebootComplete { + return fmt.Errorf("checkpoint records reboot completion but cluster observes %q", observation.Status) + } + } + + if phaseAtLeast(expectation.Phase, rotation.PhasePostRebootStable) { + if expectation.RebootIntent == nil { + return fmt.Errorf("post-reboot checkpoint is missing a reboot intent") + } + stable, err := a.postRebootStateStable(ctx, guard, *expectation.RebootIntent) + if err != nil { + return err + } + if !stable { + return fmt.Errorf("checkpoint records post-reboot stability but the cluster is not currently stable") + } + } + return nil +} + +func (a *Adapter) validateClients() error { + if a == nil || a.kube == nil || a.clusterVersions == nil || a.clusterOperators == nil || a.resources == nil || a.secretMetadata == nil { + return fmt.Errorf("rotation Kubernetes adapter is not fully configured") + } + if a.options.PollInterval <= 0 || a.options.PreflightStablePeriod < 0 || a.options.StablePeriod < 0 { + return fmt.Errorf("rotation Kubernetes adapter has invalid polling options") + } + if a.now == nil { + return fmt.Errorf("rotation Kubernetes adapter clock is not configured") + } + return nil +} + +func (a *Adapter) requireAccess(ctx context.Context, access requiredAccess) error { + review, err := a.kube.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, &authorizationv1.SelfSubjectAccessReview{ + Spec: authorizationv1.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Namespace: access.namespace, + Verb: access.verb, + Group: access.group, + Resource: access.resource, + Name: access.name, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("check %s access to %s/%s: %w", access.verb, access.group, access.resource, err) + } + if review.Status.EvaluationError != "" { + return fmt.Errorf("check %s access to %s/%s: %s", access.verb, access.group, access.resource, review.Status.EvaluationError) + } + if !review.Status.Allowed { + reason := strings.TrimSpace(review.Status.Reason) + if reason == "" { + reason = "access denied" + } + return fmt.Errorf("required %s access to %s/%s is not allowed: %s", access.verb, access.group, access.resource, reason) + } + return nil +} + +func (a *Adapter) clusterIdentity(ctx context.Context) (string, error) { + version, err := a.clusterVersions.Get(ctx, "version", metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("read cluster identity: %w", err) + } + identity := string(version.Spec.ClusterID) + if strings.TrimSpace(identity) == "" || strings.TrimSpace(identity) != identity { + return "", fmt.Errorf("ClusterVersion/version has an invalid empty cluster ID") + } + return identity, nil +} + +func (a *Adapter) clusterOperatorsStable(ctx context.Context) (bool, error) { + operators, err := a.clusterOperators.List(ctx, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("list ClusterOperators: %w", err) + } + if len(operators.Items) == 0 { + return false, nil + } + for i := range operators.Items { + operator := &operators.Items[i] + if clusterOperatorCondition(operator.Status.Conditions, configv1.OperatorAvailable) != configv1.ConditionTrue || + clusterOperatorCondition(operator.Status.Conditions, configv1.OperatorProgressing) != configv1.ConditionFalse || + clusterOperatorCondition(operator.Status.Conditions, configv1.OperatorDegraded) != configv1.ConditionFalse { + return false, nil + } + } + return true, nil +} + +func clusterOperatorCondition(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) configv1.ConditionStatus { + for _, condition := range conditions { + if condition.Type == conditionType { + return condition.Status + } + } + return configv1.ConditionUnknown +} + +func (a *Adapter) allMachineConfigPoolsStable(ctx context.Context) (bool, error) { + pools, err := a.resources.List(ctx, machineConfigPoolGVR, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("list MachineConfigPools: %w", err) + } + if len(pools.Items) == 0 { + return false, nil + } + for i := range pools.Items { + stable, err := machineConfigPoolStable(&pools.Items[i]) + if err != nil { + return false, err + } + if !stable { + return false, nil + } + } + return true, nil +} + +func machineConfigPoolStable(pool *unstructured.Unstructured) (bool, error) { + if pool == nil || pool.GetName() == "" { + return false, fmt.Errorf("MachineConfigPool has no name") + } + paused, found, err := unstructured.NestedBool(pool.Object, "spec", "paused") + if err != nil { + return false, fmt.Errorf("read MachineConfigPool %q paused state: %w", pool.GetName(), err) + } + if found && paused { + return false, nil + } + observedGeneration, found, err := unstructured.NestedInt64(pool.Object, "status", "observedGeneration") + if err != nil || !found { + return false, nil + } + if observedGeneration != pool.GetGeneration() { + return false, nil + } + specConfiguration, _, _ := unstructured.NestedString(pool.Object, "spec", "configuration", "name") + statusConfiguration, _, _ := unstructured.NestedString(pool.Object, "status", "configuration", "name") + if specConfiguration == "" || specConfiguration != statusConfiguration { + return false, nil + } + + machineCount, ok := nestedInteger(pool, "status", "machineCount") + if !ok { + return false, nil + } + updatedCount, ok := nestedInteger(pool, "status", "updatedMachineCount") + if !ok { + return false, nil + } + readyCount, ok := nestedInteger(pool, "status", "readyMachineCount") + if !ok { + return false, nil + } + unavailableCount, ok := nestedInteger(pool, "status", "unavailableMachineCount") + if !ok { + return false, nil + } + degradedCount, ok := nestedInteger(pool, "status", "degradedMachineCount") + if !ok { + return false, nil + } + if updatedCount != machineCount || readyCount != machineCount || unavailableCount != 0 || degradedCount != 0 { + return false, nil + } + conditions, found, err := unstructuredSlice(pool, "status", "conditions") + if err != nil || !found { + return false, err + } + if conditionStatus(conditions, "Updated") != string(corev1.ConditionTrue) || + conditionStatus(conditions, "Updating") != string(corev1.ConditionFalse) || + conditionStatus(conditions, "Degraded") != string(corev1.ConditionFalse) { + return false, nil + } + return true, nil +} + +func conditionStatus(conditions []any, conditionType string) string { + for _, item := range conditions { + condition, ok := item.(map[string]any) + if !ok { + continue + } + if condition["type"] == conditionType { + status, _ := condition["status"].(string) + return status + } + } + return "Unknown" +} + +func unstructuredSlice(object *unstructured.Unstructured, fields ...string) ([]any, bool, error) { + items, found, err := unstructured.NestedSlice(object.Object, fields...) + if err != nil { + return nil, false, fmt.Errorf("read %s from %q: %w", strings.Join(fields, "."), object.GetName(), err) + } + return items, found, nil +} + +func nestedInteger(object *unstructured.Unstructured, fields ...string) (int64, bool) { + value, found, err := unstructured.NestedFieldNoCopy(object.Object, fields...) + if err != nil || !found { + return 0, false + } + return integerValue(value) +} + +func integerValue(value any) (int64, bool) { + switch typed := value.(type) { + case int64: + return typed, true + case int32: + return int64(typed), true + case int: + return int64(typed), true + case float64: + if typed != float64(int64(typed)) { + return 0, false + } + return int64(typed), true + default: + return 0, false + } +} + +func phaseAtLeast(phase, required rotation.Phase) bool { + positions := make(map[rotation.Phase]int) + for index, candidate := range rotation.OrderedPhases() { + positions[candidate] = index + } + return positions[phase] >= positions[required] +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/reboot.go b/pkg/cmd/provisioning/rotation/kubeadapter/reboot.go new file mode 100644 index 0000000000..3c7f006616 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/reboot.go @@ -0,0 +1,923 @@ +package kubeadapter + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "reflect" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +const ( + rebootRecordSchemaVersion = 1 + rebootRecordDataKey = "intent.json" + rebootRecordNamePrefix = "cco-signer-key-rotation-reboot-" + maxRebootRecordDataBytes = 900 * 1024 + + rebootMachineConfigMaster = "95-cco-signer-key-rotation-reboot-master" + rebootMachineConfigWorker = "95-cco-signer-key-rotation-reboot-worker" + rebootMarkerPath = "/etc/kubernetes/cco-signer-key-rotation-reboot-id" + + managedAnnotation = "cloudcredential.openshift.io/signer-rotation-managed" + rebootIDAnnotation = "cloudcredential.openshift.io/signer-rotation-reboot-id" + rebootTargetAnnotation = "cloudcredential.openshift.io/signer-rotation-reboot-target" + rebootScopeAnnotation = "cloudcredential.openshift.io/signer-rotation-scope-id" + guardOperationAnnotation = "cloudcredential.openshift.io/signer-rotation-operation-id" + + nodeCurrentConfigAnnotation = "machineconfiguration.openshift.io/currentConfig" + nodeDesiredConfigAnnotation = "machineconfiguration.openshift.io/desiredConfig" +) + +type rebootRecord struct { + SchemaVersion int `json:"schemaVersion"` + ScopeID string `json:"scopeID"` + OperationID string `json:"operationID"` + Intent rotation.RebootIntent `json:"intent"` +} + +type rebootPoolSelection struct { + pool *unstructured.Unstructured + nodeSelector labels.Selector + machineConfigSelector labels.Selector + currentConfig string +} + +func (a *Adapter) PrepareReboot(ctx context.Context, guard rotation.RotationGuardReference, replacementKeyID string) (rotation.RebootPlan, error) { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return rotation.RebootPlan{}, err + } + if strings.TrimSpace(replacementKeyID) == "" || strings.TrimSpace(replacementKeyID) != replacementKeyID { + return rotation.RebootPlan{}, fmt.Errorf("replacement signer key ID must not be empty or contain surrounding whitespace") + } + return a.prepareRebootPlan(ctx) +} + +func (a *Adapter) prepareRebootPlan(ctx context.Context) (rotation.RebootPlan, error) { + pools, err := a.resources.List(ctx, machineConfigPoolGVR, metav1.ListOptions{}) + if err != nil { + return rotation.RebootPlan{}, fmt.Errorf("list MachineConfigPools before reboot: %w", err) + } + poolSelections := make([]rebootPoolSelection, 0, len(pools.Items)) + for i := range pools.Items { + pool := &pools.Items[i] + stable, err := machineConfigPoolStable(pool) + if err != nil { + return rotation.RebootPlan{}, err + } + if !stable { + return rotation.RebootPlan{}, fmt.Errorf("MachineConfigPool %q is not stable before reboot", pool.GetName()) + } + selection, err := newRebootPoolSelection(pool) + if err != nil { + return rotation.RebootPlan{}, err + } + poolSelections = append(poolSelections, selection) + } + targetMachineConfigLabels, targetSelectorLabels, err := rebootTargetMachineConfigLabels(pools) + if err != nil { + return rotation.RebootPlan{}, err + } + + nodes, err := a.kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return rotation.RebootPlan{}, fmt.Errorf("list nodes before reboot: %w", err) + } + if len(nodes.Items) == 0 { + return rotation.RebootPlan{}, fmt.Errorf("cluster has no nodes to reboot") + } + targets := make(map[string]struct{}, 2) + baselines := make([]rotation.NodeRebootBaseline, 0, len(nodes.Items)) + observedPoolCounts := make(map[string]int64, len(poolSelections)) + for i := range nodes.Items { + node := &nodes.Items[i] + if !nodeReady(node) { + return rotation.RebootPlan{}, fmt.Errorf("node %q is not Ready before reboot", node.Name) + } + if strings.TrimSpace(node.Status.NodeInfo.BootID) == "" { + return rotation.RebootPlan{}, fmt.Errorf("node %q has no boot ID", node.Name) + } + currentConfig := node.Annotations[nodeCurrentConfigAnnotation] + desiredConfig := node.Annotations[nodeDesiredConfigAnnotation] + if currentConfig == "" || desiredConfig == "" || currentConfig != desiredConfig { + return rotation.RebootPlan{}, fmt.Errorf("node %q is not stably managed by the Machine Config Operator", node.Name) + } + pool, err := effectiveMachineConfigPoolForNode(node, currentConfig, poolSelections) + if err != nil { + return rotation.RebootPlan{}, err + } + target, err := rebootTargetForPool(node.Name, pool, targetMachineConfigLabels) + if err != nil { + return rotation.RebootPlan{}, err + } + observedPoolCounts[pool.pool.GetName()]++ + targets[target] = struct{}{} + baselines = append(baselines, rotation.NodeRebootBaseline{Target: target, Node: node.Name, BootID: node.Status.NodeInfo.BootID}) + } + for _, pool := range poolSelections { + expectedCount, ok := nestedInteger(pool.pool, "status", "machineCount") + if !ok { + return rotation.RebootPlan{}, fmt.Errorf("MachineConfigPool %q has no machine count", pool.pool.GetName()) + } + if observedPoolCounts[pool.pool.GetName()] != expectedCount { + return rotation.RebootPlan{}, fmt.Errorf("MachineConfigPool %q reports %d nodes but %d were resolved from the stable node snapshot", pool.pool.GetName(), expectedCount, observedPoolCounts[pool.pool.GetName()]) + } + } + + orderedTargets := make([]string, 0, len(targets)) + for target := range targets { + orderedTargets = append(orderedTargets, target) + } + slices.Sort(orderedTargets) + slices.SortFunc(baselines, func(left, right rotation.NodeRebootBaseline) int { + if comparison := strings.Compare(left.Target, right.Target); comparison != 0 { + return comparison + } + return strings.Compare(left.Node, right.Node) + }) + for _, target := range orderedTargets { + if err := a.validateExistingRebootMachineConfig(ctx, target, targetSelectorLabels[target]); err != nil { + return rotation.RebootPlan{}, err + } + } + return rotation.RebootPlan{Targets: orderedTargets, Baselines: baselines}, nil +} + +func (a *Adapter) ObserveReboot(ctx context.Context, guard rotation.RotationGuardReference, operationID string) (rotation.RebootObservation, error) { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return rotation.RebootObservation{}, err + } + if err := validateRebootID(operationID); err != nil { + return rotation.RebootObservation{}, err + } + configMap, err := a.kube.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace).Get(ctx, rebootRecordName(operationID), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return rotation.RebootObservation{Status: rotation.RebootNotStarted}, nil + } + if err != nil { + return rotation.RebootObservation{}, fmt.Errorf("read canonical reboot record: %w", err) + } + record, err := decodeRebootRecord(configMap, guard, operationID) + if err != nil { + return rotation.RebootObservation{}, err + } + intent := cloneRebootIntent(record.Intent) + observation := rotation.RebootObservation{Status: rotation.RebootInProgress, CanonicalIntent: &intent} + + for _, target := range intent.Targets { + machineConfig, err := a.resources.Get(ctx, machineConfigGVR, rebootMachineConfigName(target), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return observation, nil + } + if err != nil { + return rotation.RebootObservation{}, fmt.Errorf("read reboot MachineConfig for target %q: %w", target, err) + } + pool, err := a.resources.Get(ctx, machineConfigPoolGVR, target, metav1.GetOptions{}) + if err != nil { + return rotation.RebootObservation{}, fmt.Errorf("read MachineConfigPool %q during reboot observation: %w", target, err) + } + selectorLabels, err := machineConfigSelectorLabels(pool) + if err != nil { + return rotation.RebootObservation{}, err + } + if err := validateRebootMachineConfig(machineConfig, target, intent.ID, selectorLabels); err != nil { + return rotation.RebootObservation{}, err + } + } + + renderedConfigs := make(map[string]*unstructured.Unstructured) + for _, baseline := range intent.Baselines { + node, err := a.kube.CoreV1().Nodes().Get(ctx, baseline.Node, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + // Node replacement or removal proves the pre-intent instance is no + // longer running. This matches the OpenShift node-reboot waiter and + // keeps an immutable baseline from becoming an unrecoverable dead end. + continue + } + if err != nil { + return rotation.RebootObservation{}, fmt.Errorf("read reboot node %q: %w", baseline.Node, err) + } + if node.Status.NodeInfo.BootID == "" { + return rotation.RebootObservation{}, fmt.Errorf("reboot node %q has no boot ID", baseline.Node) + } + if node.Status.NodeInfo.BootID == baseline.BootID { + return observation, nil + } + currentConfig := node.Annotations[nodeCurrentConfigAnnotation] + desiredConfig := node.Annotations[nodeDesiredConfigAnnotation] + if currentConfig == "" || currentConfig != desiredConfig { + return observation, nil + } + rendered := renderedConfigs[currentConfig] + if rendered == nil { + rendered, err = a.resources.Get(ctx, machineConfigGVR, currentConfig, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return observation, nil + } + if err != nil { + return rotation.RebootObservation{}, fmt.Errorf("read node %q current MachineConfig %q: %w", baseline.Node, currentConfig, err) + } + renderedConfigs[currentConfig] = rendered + } + if !machineConfigHasRebootMarker(rendered, intent.ID) { + return observation, nil + } + } + + observation.Status = rotation.RebootComplete + return observation, nil +} + +func (a *Adapter) RequestReboot(ctx context.Context, guard rotation.RotationGuardReference, intent rotation.RebootIntent) (rotation.EffectOutcome, error) { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return rotation.EffectNotApplied, err + } + if err := validateRebootIntent(intent); err != nil { + return rotation.EffectNotApplied, err + } + recordApplied, recordOutcome, err := a.ensureRebootRecord(ctx, guard, intent) + if err != nil { + return recordOutcome, err + } + if !recordApplied { + return rotation.EffectUnknown, fmt.Errorf("canonical reboot record is not observable") + } + + targets := append([]string(nil), intent.Targets...) + slices.Sort(targets) + for _, target := range targets { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return rotation.EffectUnknown, fmt.Errorf("reconcile reboot target %q after recording canonical intent: %w", target, err) + } + if _, err := a.ensureRebootMachineConfig(ctx, target, intent.ID); err != nil { + return rotation.EffectUnknown, fmt.Errorf("reconcile reboot target %q: %w", target, err) + } + } + return rotation.EffectSubmitted, nil +} + +func (a *Adapter) WaitForReboot(ctx context.Context, guard rotation.RotationGuardReference, intent rotation.RebootIntent) error { + if err := validateRebootIntent(intent); err != nil { + return err + } + return a.waitUntil(ctx, func(ctx context.Context) (bool, error) { + observation, err := a.ObserveReboot(ctx, guard, intent.ID) + if err != nil { + return false, err + } + if observation.Status == rotation.RebootNotStarted || observation.CanonicalIntent == nil { + return false, fmt.Errorf("canonical reboot record %q disappeared while waiting", intent.ID) + } + if !reflect.DeepEqual(*observation.CanonicalIntent, intent) { + return false, fmt.Errorf("canonical reboot record %q differs from the requested intent", intent.ID) + } + if observation.Status == rotation.RebootComplete { + return true, nil + } + healthy, err := a.affectedMachineConfigPoolsHealthy(ctx, intent) + if err != nil { + return false, err + } + if !healthy { + return false, fmt.Errorf("an affected MachineConfigPool became paused or degraded during reboot") + } + return false, nil + }) +} + +func (a *Adapter) WaitForPostRebootStable(ctx context.Context, guard rotation.RotationGuardReference, intent rotation.RebootIntent) error { + if err := validateRebootIntent(intent); err != nil { + return err + } + return a.waitForContinuousStability(ctx, a.options.StablePeriod, func(ctx context.Context) (bool, error) { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return false, err + } + return a.postRebootStateStable(ctx, guard, intent) + }) +} + +func (a *Adapter) postRebootStateStable(ctx context.Context, guard rotation.RotationGuardReference, intent rotation.RebootIntent) (bool, error) { + observation, err := a.ObserveReboot(ctx, guard, intent.ID) + if err != nil { + return false, err + } + if observation.Status != rotation.RebootComplete || observation.CanonicalIntent == nil || !reflect.DeepEqual(*observation.CanonicalIntent, intent) { + return false, nil + } + stable, err := a.affectedMachineConfigPoolsStable(ctx, intent) + if err != nil || !stable { + return false, err + } + nodesStable, err := a.currentNodesStableForReboot(ctx, intent) + if err != nil || !nodesStable { + return false, err + } + return a.clusterOperatorsStable(ctx) +} + +func (a *Adapter) currentNodesStableForReboot(ctx context.Context, intent rotation.RebootIntent) (bool, error) { + pools, err := a.resources.List(ctx, machineConfigPoolGVR, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("list MachineConfigPools for post-reboot node validation: %w", err) + } + targetMachineConfigLabels, _, err := rebootTargetMachineConfigLabels(pools) + if err != nil { + return false, err + } + poolSelections := make([]rebootPoolSelection, 0, len(pools.Items)) + for i := range pools.Items { + selection, err := newRebootPoolSelection(&pools.Items[i]) + if err != nil { + return false, err + } + poolSelections = append(poolSelections, selection) + } + nodes, err := a.kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("list post-reboot nodes: %w", err) + } + if len(nodes.Items) == 0 { + return false, nil + } + targets := make(map[string]struct{}, len(intent.Targets)) + for _, target := range intent.Targets { + targets[target] = struct{}{} + } + renderedConfigs := make(map[string]*unstructured.Unstructured) + for i := range nodes.Items { + node := &nodes.Items[i] + if !nodeReady(node) { + return false, nil + } + currentConfig := node.Annotations[nodeCurrentConfigAnnotation] + desiredConfig := node.Annotations[nodeDesiredConfigAnnotation] + if currentConfig == "" || currentConfig != desiredConfig { + return false, nil + } + pool, err := effectiveMachineConfigPoolForNode(node, currentConfig, poolSelections) + if err != nil { + return false, err + } + target, err := rebootTargetForPool(node.Name, pool, targetMachineConfigLabels) + if err != nil { + return false, err + } + if _, expected := targets[target]; !expected { + return false, fmt.Errorf("post-reboot node %q belongs to target %q outside the canonical intent", node.Name, target) + } + rendered := renderedConfigs[currentConfig] + if rendered == nil { + rendered, err = a.resources.Get(ctx, machineConfigGVR, currentConfig, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("read post-reboot node %q current MachineConfig %q: %w", node.Name, currentConfig, err) + } + renderedConfigs[currentConfig] = rendered + } + if !machineConfigHasRebootMarker(rendered, intent.ID) { + return false, nil + } + } + return true, nil +} + +func (a *Adapter) ensureRebootRecord(ctx context.Context, guard rotation.RotationGuardReference, intent rotation.RebootIntent) (bool, rotation.EffectOutcome, error) { + client := a.kube.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace) + name := rebootRecordName(intent.ID) + current, err := client.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + record, decodeErr := decodeRebootRecord(current, guard, intent.ID) + if decodeErr != nil { + return false, rotation.EffectNotApplied, decodeErr + } + if !reflect.DeepEqual(record.Intent, intent) { + return false, rotation.EffectNotApplied, fmt.Errorf("canonical reboot record %q contains a different intent", name) + } + return true, rotation.EffectSubmitted, nil + } + if !apierrors.IsNotFound(err) { + return false, rotation.EffectNotApplied, fmt.Errorf("read canonical reboot record before creation: %w", err) + } + currentPlan, err := a.prepareRebootPlan(ctx) + if err != nil { + return false, rotation.EffectNotApplied, fmt.Errorf("revalidate reboot plan before recording canonical intent: %w", err) + } + if !reflect.DeepEqual(currentPlan.Targets, intent.Targets) || !reflect.DeepEqual(currentPlan.Baselines, intent.Baselines) { + return false, rotation.EffectNotApplied, fmt.Errorf("current reboot plan differs from the checkpointed intent") + } + + configMap, err := newRebootRecordConfigMap(guard, intent) + if err != nil { + return false, rotation.EffectNotApplied, err + } + _, createErr := client.Create(ctx, configMap, metav1.CreateOptions{}) + if createErr == nil { + return true, rotation.EffectSubmitted, nil + } + createOutcome := mutationOutcome(createErr) + observed, observeErr := client.Get(ctx, name, metav1.GetOptions{}) + if observeErr == nil { + record, decodeErr := decodeRebootRecord(observed, guard, intent.ID) + if decodeErr != nil { + return false, createOutcome, errors.Join(createErr, decodeErr) + } + if reflect.DeepEqual(record.Intent, intent) { + return true, rotation.EffectSubmitted, nil + } + return false, createOutcome, errors.Join(createErr, fmt.Errorf("canonical reboot record %q contains a different intent", name)) + } + return false, createOutcome, errors.Join(createErr, observeErr) +} + +func newRebootRecordConfigMap(guard rotation.RotationGuardReference, intent rotation.RebootIntent) (*corev1.ConfigMap, error) { + record := rebootRecord{SchemaVersion: rebootRecordSchemaVersion, ScopeID: guard.ScopeID, OperationID: guard.OperationID, Intent: cloneRebootIntent(intent)} + encoded, err := json.Marshal(record) + if err != nil { + return nil, fmt.Errorf("encode canonical reboot record: %w", err) + } + if len(encoded) > maxRebootRecordDataBytes { + return nil, fmt.Errorf("canonical reboot record is too large: %d bytes", len(encoded)) + } + immutable := true + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cloudCredentialOperatorNamespace, + Name: rebootRecordName(intent.ID), + Labels: map[string]string{managedByLabelKey: managedByLabelValue}, + Annotations: map[string]string{ + managedAnnotation: "true", + rebootScopeAnnotation: guard.ScopeID, + guardOperationAnnotation: guard.OperationID, + rebootIDAnnotation: intent.ID, + }, + }, + Immutable: &immutable, + Data: map[string]string{rebootRecordDataKey: string(encoded)}, + }, nil +} + +func decodeRebootRecord(configMap *corev1.ConfigMap, guard rotation.RotationGuardReference, operationID string) (rebootRecord, error) { + var record rebootRecord + if configMap == nil || configMap.Namespace != cloudCredentialOperatorNamespace || configMap.Name != rebootRecordName(operationID) { + return record, fmt.Errorf("unexpected canonical reboot record identity") + } + if configMap.DeletionTimestamp != nil { + return record, fmt.Errorf("canonical reboot record %q is being deleted", configMap.Name) + } + if configMap.Immutable == nil || !*configMap.Immutable { + return record, fmt.Errorf("canonical reboot record %q is not immutable", configMap.Name) + } + if len(configMap.BinaryData) != 0 || len(configMap.Data) != 1 { + return record, fmt.Errorf("canonical reboot record %q has unexpected data fields", configMap.Name) + } + raw, exists := configMap.Data[rebootRecordDataKey] + if !exists { + return record, fmt.Errorf("canonical reboot record %q is missing %q", configMap.Name, rebootRecordDataKey) + } + decoder := json.NewDecoder(bytes.NewBufferString(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&record); err != nil { + return record, fmt.Errorf("decode canonical reboot record %q: %w", configMap.Name, err) + } + if err := ensureJSONEOF(decoder); err != nil { + return record, fmt.Errorf("decode canonical reboot record %q: %w", configMap.Name, err) + } + if record.SchemaVersion != rebootRecordSchemaVersion { + return record, fmt.Errorf("unsupported canonical reboot record schema version %d", record.SchemaVersion) + } + if record.ScopeID != guard.ScopeID || record.OperationID != guard.OperationID || record.Intent.ID != operationID { + return record, fmt.Errorf("canonical reboot record %q is bound to different rotation evidence", configMap.Name) + } + if err := validateRebootIntent(record.Intent); err != nil { + return record, fmt.Errorf("invalid canonical reboot record %q: %w", configMap.Name, err) + } + return record, nil +} + +func (a *Adapter) ensureRebootMachineConfig(ctx context.Context, target, rebootID string) (bool, error) { + pool, err := a.resources.Get(ctx, machineConfigPoolGVR, target, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("read MachineConfigPool %q: %w", target, err) + } + selectorLabels, err := machineConfigSelectorLabels(pool) + if err != nil { + return false, err + } + name := rebootMachineConfigName(target) + current, err := a.resources.Get(ctx, machineConfigGVR, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + desired := desiredRebootMachineConfig(nil, name, target, rebootID, selectorLabels) + _, createErr := a.resources.Create(ctx, machineConfigGVR, desired, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(createErr) { + return a.ensureRebootMachineConfig(ctx, target, rebootID) + } + return createErr == nil, createErr + } + if err != nil { + return false, fmt.Errorf("read reboot MachineConfig %q: %w", name, err) + } + currentID, err := validateManagedRebootMachineConfig(current, target, selectorLabels) + if err != nil { + return false, err + } + if currentID == rebootID { + return false, nil + } + desired := desiredRebootMachineConfig(current, name, target, rebootID, selectorLabels) + _, updateErr := a.resources.Update(ctx, machineConfigGVR, desired, metav1.UpdateOptions{}) + return updateErr == nil, updateErr +} + +func (a *Adapter) validateExistingRebootMachineConfig(ctx context.Context, target string, selectorLabels map[string]string) error { + name := rebootMachineConfigName(target) + current, err := a.resources.Get(ctx, machineConfigGVR, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("read reserved reboot MachineConfig %q during preflight: %w", name, err) + } + if _, err := validateManagedRebootMachineConfig(current, target, selectorLabels); err != nil { + return fmt.Errorf("validate reserved reboot MachineConfig %q during preflight: %w", name, err) + } + return nil +} + +func validateManagedRebootMachineConfig(machineConfig *unstructured.Unstructured, target string, selectorLabels map[string]string) (string, error) { + name := rebootMachineConfigName(target) + if machineConfig.GetAnnotations()[managedAnnotation] != "true" || machineConfig.GetAnnotations()[rebootTargetAnnotation] != target { + return "", fmt.Errorf("MachineConfig %q exists without expected rotation ownership", name) + } + currentID := machineConfig.GetAnnotations()[rebootIDAnnotation] + if err := validateRebootID(currentID); err != nil { + return "", fmt.Errorf("MachineConfig %q has invalid prior reboot identity: %w", name, err) + } + if err := validateRebootMachineConfig(machineConfig, target, currentID, selectorLabels); err != nil { + return "", err + } + return currentID, nil +} + +func desiredRebootMachineConfig(current *unstructured.Unstructured, name, target, rebootID string, selectorLabels map[string]string) *unstructured.Unstructured { + object := &unstructured.Unstructured{} + if current != nil { + object = current.DeepCopy() + } + object.SetAPIVersion("machineconfiguration.openshift.io/v1") + object.SetKind("MachineConfig") + object.SetName(name) + labelsCopy := make(map[string]string, len(selectorLabels)+1) + for key, value := range selectorLabels { + labelsCopy[key] = value + } + labelsCopy[managedByLabelKey] = managedByLabelValue + object.SetLabels(labelsCopy) + object.SetAnnotations(map[string]string{ + managedAnnotation: "true", + rebootIDAnnotation: rebootID, + rebootTargetAnnotation: target, + }) + object.Object["spec"] = map[string]any{ + "config": map[string]any{ + "ignition": map[string]any{"version": "3.1.0"}, + "storage": map[string]any{ + "files": []any{map[string]any{ + "path": rebootMarkerPath, + "mode": int64(0644), + "overwrite": true, + "contents": map[string]any{ + "source": rebootMarkerSource(rebootID), + }, + }}, + }, + }, + } + return object +} + +func validateRebootMachineConfig(machineConfig *unstructured.Unstructured, target, rebootID string, selectorLabels map[string]string) error { + expected := desiredRebootMachineConfig(nil, rebootMachineConfigName(target), target, rebootID, selectorLabels) + if machineConfig.GetAPIVersion() != expected.GetAPIVersion() || machineConfig.GetKind() != expected.GetKind() || machineConfig.GetName() != expected.GetName() { + return fmt.Errorf("MachineConfig %q has an unexpected identity for reboot target %q", machineConfig.GetName(), target) + } + if !reflect.DeepEqual(machineConfig.GetLabels(), expected.GetLabels()) || !reflect.DeepEqual(machineConfig.GetAnnotations(), expected.GetAnnotations()) { + return fmt.Errorf("MachineConfig %q does not contain the exact reboot ownership metadata", machineConfig.GetName()) + } + if !reflect.DeepEqual(machineConfig.Object["spec"], expected.Object["spec"]) { + return fmt.Errorf("MachineConfig %q does not contain the exact reboot specification", machineConfig.GetName()) + } + return nil +} + +func machineConfigHasRebootMarker(machineConfig *unstructured.Unstructured, rebootID string) bool { + files, found, err := unstructured.NestedSlice(machineConfig.Object, "spec", "config", "storage", "files") + if err != nil || !found { + return false + } + for _, item := range files { + file, ok := item.(map[string]any) + if !ok || file["path"] != rebootMarkerPath { + continue + } + contents, ok := file["contents"].(map[string]any) + return ok && contents["source"] == rebootMarkerSource(rebootID) + } + return false +} + +func machineConfigSelectorLabels(pool *unstructured.Unstructured) (map[string]string, error) { + selector, found, err := unstructured.NestedMap(pool.Object, "spec", "machineConfigSelector") + if err != nil || !found { + return nil, fmt.Errorf("MachineConfigPool %q has no machineConfigSelector", pool.GetName()) + } + if expressions, exists := selector["matchExpressions"]; exists { + if items, ok := expressions.([]any); !ok || len(items) != 0 { + return nil, fmt.Errorf("MachineConfigPool %q uses unsupported machineConfigSelector matchExpressions", pool.GetName()) + } + } + labelsValue, exists := selector["matchLabels"] + if !exists { + return nil, fmt.Errorf("MachineConfigPool %q has no machineConfigSelector matchLabels", pool.GetName()) + } + labelsMap, ok := labelsValue.(map[string]any) + if !ok || len(labelsMap) == 0 { + return nil, fmt.Errorf("MachineConfigPool %q has invalid machineConfigSelector matchLabels", pool.GetName()) + } + result := make(map[string]string, len(labelsMap)) + for key, value := range labelsMap { + text, ok := value.(string) + if !ok || key == "" || text == "" { + return nil, fmt.Errorf("MachineConfigPool %q has invalid machineConfigSelector matchLabels", pool.GetName()) + } + result[key] = text + } + return result, nil +} + +func machineConfigPoolLabelSelector(pool *unstructured.Unstructured, field string) (labels.Selector, error) { + selectorMap, found, err := unstructured.NestedMap(pool.Object, "spec", field) + if err != nil || !found { + return nil, fmt.Errorf("MachineConfigPool %q has no %s", pool.GetName(), field) + } + encoded, err := json.Marshal(selectorMap) + if err != nil { + return nil, fmt.Errorf("encode MachineConfigPool %q %s: %w", pool.GetName(), field, err) + } + var selectorSpec metav1.LabelSelector + if err := json.Unmarshal(encoded, &selectorSpec); err != nil { + return nil, fmt.Errorf("decode MachineConfigPool %q %s: %w", pool.GetName(), field, err) + } + selector, err := metav1.LabelSelectorAsSelector(&selectorSpec) + if err != nil { + return nil, fmt.Errorf("parse MachineConfigPool %q %s: %w", pool.GetName(), field, err) + } + if selector.Empty() { + return nil, fmt.Errorf("MachineConfigPool %q has an empty %s", pool.GetName(), field) + } + return selector, nil +} + +func newRebootPoolSelection(pool *unstructured.Unstructured) (rebootPoolSelection, error) { + nodeSelector, err := machineConfigPoolLabelSelector(pool, "nodeSelector") + if err != nil { + return rebootPoolSelection{}, err + } + machineConfigSelector, err := machineConfigPoolLabelSelector(pool, "machineConfigSelector") + if err != nil { + return rebootPoolSelection{}, err + } + currentConfig, found, err := unstructured.NestedString(pool.Object, "status", "configuration", "name") + if err != nil || !found || currentConfig == "" { + return rebootPoolSelection{}, fmt.Errorf("MachineConfigPool %q has no current rendered configuration", pool.GetName()) + } + return rebootPoolSelection{ + pool: pool.DeepCopy(), + nodeSelector: nodeSelector, + machineConfigSelector: machineConfigSelector, + currentConfig: currentConfig, + }, nil +} + +func rebootTargetMachineConfigLabels(pools *unstructured.UnstructuredList) (map[string]labels.Set, map[string]map[string]string, error) { + basePools := make(map[string]*unstructured.Unstructured, 2) + for i := range pools.Items { + pool := &pools.Items[i] + if pool.GetName() == "master" || pool.GetName() == "worker" { + basePools[pool.GetName()] = pool + } + } + machineConfigLabels := make(map[string]labels.Set, 2) + selectorLabelsByTarget := make(map[string]map[string]string, 2) + for _, target := range []string{"master", "worker"} { + pool := basePools[target] + if pool == nil { + return nil, nil, fmt.Errorf("required MachineConfigPool %q was not found", target) + } + selectorLabels, err := machineConfigSelectorLabels(pool) + if err != nil { + return nil, nil, err + } + selectorLabelsByTarget[target] = selectorLabels + labelsForTarget := labels.Set{} + for key, value := range selectorLabels { + labelsForTarget[key] = value + } + labelsForTarget[managedByLabelKey] = managedByLabelValue + machineConfigLabels[target] = labelsForTarget + } + return machineConfigLabels, selectorLabelsByTarget, nil +} + +func effectiveMachineConfigPoolForNode(node *corev1.Node, currentConfig string, pools []rebootPoolSelection) (rebootPoolSelection, error) { + matches := make([]rebootPoolSelection, 0, 1) + for _, pool := range pools { + if pool.currentConfig == currentConfig && pool.nodeSelector.Matches(labels.Set(node.Labels)) { + matches = append(matches, pool) + } + } + if len(matches) != 1 { + names := make([]string, 0, len(matches)) + for _, match := range matches { + names = append(names, match.pool.GetName()) + } + slices.Sort(names) + return rebootPoolSelection{}, fmt.Errorf("node %q with current MachineConfig %q resolves to %d MachineConfigPools %v; exactly one is required", node.Name, currentConfig, len(matches), names) + } + return matches[0], nil +} + +func rebootTargetForPool(nodeName string, pool rebootPoolSelection, targetMachineConfigLabels map[string]labels.Set) (string, error) { + matches := make([]string, 0, 2) + for _, target := range []string{"master", "worker"} { + if pool.machineConfigSelector.Matches(targetMachineConfigLabels[target]) { + matches = append(matches, target) + } + } + if len(matches) != 1 { + return "", fmt.Errorf("node %q is managed by MachineConfigPool %q, whose machineConfigSelector matches %d supported reboot targets %v; exactly one of master or worker is required", nodeName, pool.pool.GetName(), len(matches), matches) + } + return matches[0], nil +} + +func (a *Adapter) affectedMachineConfigPoolsHealthy(ctx context.Context, intent rotation.RebootIntent) (bool, error) { + pools, err := a.affectedMachineConfigPools(ctx, intent) + if err != nil { + return false, err + } + for _, pool := range pools { + paused, _, err := unstructured.NestedBool(pool.Object, "spec", "paused") + if err != nil { + return false, err + } + degraded, ok := nestedInteger(pool, "status", "degradedMachineCount") + if paused || !ok || degraded != 0 { + return false, nil + } + conditions, found, err := unstructuredSlice(pool, "status", "conditions") + if err != nil || !found || conditionStatus(conditions, "Degraded") != string(corev1.ConditionFalse) { + return false, err + } + } + return true, nil +} + +func (a *Adapter) affectedMachineConfigPoolsStable(ctx context.Context, intent rotation.RebootIntent) (bool, error) { + pools, err := a.affectedMachineConfigPools(ctx, intent) + if err != nil { + return false, err + } + for _, pool := range pools { + stable, err := machineConfigPoolStable(pool) + if err != nil || !stable { + return false, err + } + } + return true, nil +} + +func (a *Adapter) affectedMachineConfigPools(ctx context.Context, intent rotation.RebootIntent) ([]*unstructured.Unstructured, error) { + machineConfigLabels := make([]labels.Set, 0, len(intent.Targets)) + for _, target := range intent.Targets { + machineConfig, err := a.resources.Get(ctx, machineConfigGVR, rebootMachineConfigName(target), metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("read reboot MachineConfig for target %q: %w", target, err) + } + machineConfigLabels = append(machineConfigLabels, labels.Set(machineConfig.GetLabels())) + } + pools, err := a.resources.List(ctx, machineConfigPoolGVR, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("list affected MachineConfigPools: %w", err) + } + affected := make([]*unstructured.Unstructured, 0, len(pools.Items)) + for i := range pools.Items { + pool := &pools.Items[i] + selector, err := machineConfigPoolLabelSelector(pool, "machineConfigSelector") + if err != nil { + return nil, err + } + for _, machineLabels := range machineConfigLabels { + if selector.Matches(machineLabels) { + affected = append(affected, pool.DeepCopy()) + break + } + } + } + if len(affected) == 0 { + return nil, fmt.Errorf("no MachineConfigPool selects the reboot MachineConfigs") + } + return affected, nil +} + +func validateRebootIntent(intent rotation.RebootIntent) error { + if err := validateRebootID(intent.ID); err != nil { + return err + } + if len(intent.Targets) == 0 || len(intent.Baselines) == 0 { + return fmt.Errorf("reboot intent must include targets and node baselines") + } + targets := make(map[string]struct{}, len(intent.Targets)) + for _, target := range intent.Targets { + if target != "master" && target != "worker" { + return fmt.Errorf("unsupported reboot target %q", target) + } + if _, duplicate := targets[target]; duplicate { + return fmt.Errorf("duplicate reboot target %q", target) + } + targets[target] = struct{}{} + } + counts := make(map[string]int, len(targets)) + nodes := make(map[string]struct{}, len(intent.Baselines)) + for _, baseline := range intent.Baselines { + if _, exists := targets[baseline.Target]; !exists || baseline.Node == "" || baseline.BootID == "" { + return fmt.Errorf("invalid reboot baseline for node %q", baseline.Node) + } + if _, duplicate := nodes[baseline.Node]; duplicate { + return fmt.Errorf("duplicate reboot baseline for node %q", baseline.Node) + } + nodes[baseline.Node] = struct{}{} + counts[baseline.Target]++ + } + for target := range targets { + if counts[target] == 0 { + return fmt.Errorf("reboot target %q has no node baseline", target) + } + } + return nil +} + +func validateRebootID(operationID string) error { + const prefix = "signer-rotation-" + if !strings.HasPrefix(operationID, prefix) { + return fmt.Errorf("invalid reboot operation ID") + } + if err := validateSHA256(strings.TrimPrefix(operationID, prefix)); err != nil { + return fmt.Errorf("invalid reboot operation ID: %w", err) + } + return nil +} + +func rebootRecordName(operationID string) string { + return rebootRecordNamePrefix + strings.TrimPrefix(operationID, "signer-rotation-") +} + +func rebootMachineConfigName(target string) string { + if target == "master" { + return rebootMachineConfigMaster + } + return rebootMachineConfigWorker +} + +func rebootMarkerSource(rebootID string) string { + return "data:," + url.PathEscape(rebootID+"\n") +} + +func cloneRebootIntent(intent rotation.RebootIntent) rotation.RebootIntent { + return rotation.RebootIntent{ + ID: intent.ID, + Targets: append([]string(nil), intent.Targets...), + Baselines: append([]rotation.NodeRebootBaseline(nil), intent.Baselines...), + } +} + +func nodeReady(node *corev1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go b/pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go new file mode 100644 index 0000000000..b2f155fc40 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go @@ -0,0 +1,740 @@ +package kubeadapter + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + configv1 "github.com/openshift/api/config/v1" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +func TestPreflightUsesClusterIDAndChecksEveryRequiredAccess(t *testing.T) { + client := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cloudCredentialOperatorNamespace}}, + readyNode("master-0", "master-boot"), + readyNode("worker-0", "worker-boot"), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kubeAPIServerNamespace, + Name: publicSignerCMName, + UID: types.UID("public-cm-uid"), + ResourceVersion: "8", + }, + Data: map[string]string{"service-account-001.pub": "public"}, + }, + ) + master, err := client.CoreV1().Nodes().Get(context.Background(), "master-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + master.Labels = map[string]string{"node-role.kubernetes.io/master": ""} + if _, err := client.CoreV1().Nodes().Update(context.Background(), master, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + client.Fake.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SelfSubjectAccessReview).DeepCopy() + review.Status.Allowed = true + return true, review, nil + }) + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 1), + testKubeAPIServer(7), + ) + clusterID := configv1.ClusterID("a58a4337-2d99-41ed-b009-33b62d854224") + metadata := &metav1.PartialObjectMetadata{ObjectMeta: metav1.ObjectMeta{ + Namespace: kubeAPIServerOperatorNamespace, + Name: nextSignerSecretName, + UID: types.UID("next-signer-uid"), + ResourceVersion: "9", + }} + adapter := newAdapter( + client, + staticClusterVersionGetter{version: &configv1.ClusterVersion{Spec: configv1.ClusterVersionSpec{ClusterID: clusterID}}}, + staticClusterOperatorLister{operators: stableClusterOperators()}, + resources, + staticSecretMetadataGetter{metadata: metadata}, + testOptions(), + ) + + preflight, err := adapter.Preflight(context.Background()) + if err != nil { + t.Fatalf("Preflight() returned unexpected error: %v", err) + } + if preflight.ClusterIdentity != string(clusterID) { + t.Fatalf("cluster identity = %q, want %q", preflight.ClusterIdentity, clusterID) + } + accessChecks := 0 + for _, action := range client.Actions() { + if action.Matches("create", "selfsubjectaccessreviews") { + accessChecks++ + } + } + if accessChecks != len(preflightAccess) { + t.Fatalf("SelfSubjectAccessReview calls = %d, want %d", accessChecks, len(preflightAccess)) + } +} + +func TestPreflightFailsClosedWhenRequiredAccessIsDenied(t *testing.T) { + client := fake.NewSimpleClientset() + client.Fake.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SelfSubjectAccessReview).DeepCopy() + review.Status.Allowed = false + review.Status.Reason = "policy denied" + return true, review, nil + }) + adapter := newAdapter( + client, + staticClusterVersionGetter{}, + staticClusterOperatorLister{}, + newMemoryResourceClient(), + staticSecretMetadataGetter{}, + testOptions(), + ) + + _, err := adapter.Preflight(context.Background()) + if err == nil || !strings.Contains(err.Error(), "policy denied") { + t.Fatalf("Preflight() error = %v, want access denial", err) + } +} + +func TestMachineConfigPoolStabilityRejectsUpdatingPool(t *testing.T) { + pool := stableMachineConfigPool("worker", 2) + _ = unstructured.SetNestedSlice(pool.Object, []any{ + map[string]any{"type": "Updated", "status": "False"}, + map[string]any{"type": "Updating", "status": "True"}, + map[string]any{"type": "Degraded", "status": "False"}, + }, "status", "conditions") + stable, err := machineConfigPoolStable(pool) + if err != nil { + t.Fatalf("machineConfigPoolStable() returned unexpected error: %v", err) + } + if stable { + t.Fatal("machineConfigPoolStable() = true for an updating pool") + } +} + +func TestPrepareRebootRejectsNodeOutsideStableMCOManagement(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + node := readyNode("unmanaged-0", "boot-before") + node.Annotations = nil + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + node, + ) + adapter := &Adapter{ + kube: client, + resources: newMemoryResourceClient( + stableMachineConfigPool("master", 0), + stableMachineConfigPool("worker", 1), + ), + } + + _, err := adapter.PrepareReboot(ctx, guard, "replacement-key-id") + if err == nil || !strings.Contains(err.Error(), "not stably managed") { + t.Fatalf("PrepareReboot() error = %v, want unmanaged-node rejection", err) + } +} + +func TestPrepareRebootSupportsCustomPoolThatInheritsWorkerMachineConfigs(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + node := readyNode("worker-infra-0", "boot-before") + delete(node.Labels, "node-role.kubernetes.io/worker") + node.Labels["node-role.kubernetes.io/infra"] = "" + node.Annotations[nodeCurrentConfigAnnotation] = "rendered-infra-old" + node.Annotations[nodeDesiredConfigAnnotation] = "rendered-infra-old" + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + node, + ) + infraPool := stableMachineConfigPool("infra", 1) + _ = unstructured.SetNestedMap(infraPool.Object, map[string]any{ + "matchExpressions": []any{map[string]any{ + "key": "machineconfiguration.openshift.io/role", + "operator": "In", + "values": []any{"worker", "infra"}, + }}, + }, "spec", "machineConfigSelector") + adapter := &Adapter{ + kube: client, + resources: newMemoryResourceClient( + stableMachineConfigPool("master", 0), + stableMachineConfigPool("worker", 0), + infraPool, + ), + } + + plan, err := adapter.PrepareReboot(ctx, guard, "replacement-key-id") + if err != nil { + t.Fatalf("PrepareReboot() returned unexpected error: %v", err) + } + want := rotation.RebootPlan{ + Targets: []string{"worker"}, + Baselines: []rotation.NodeRebootBaseline{{Target: "worker", Node: node.Name, BootID: "boot-before"}}, + } + if !reflect.DeepEqual(plan, want) { + t.Fatalf("PrepareReboot() = %#v, want %#v", plan, want) + } +} + +func TestPrepareRebootRejectsCustomPoolThatDoesNotInheritTargetMachineConfigs(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + node := readyNode("worker-infra-0", "boot-before") + delete(node.Labels, "node-role.kubernetes.io/worker") + node.Labels["node-role.kubernetes.io/infra"] = "" + node.Annotations[nodeCurrentConfigAnnotation] = "rendered-infra-old" + node.Annotations[nodeDesiredConfigAnnotation] = "rendered-infra-old" + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + node, + ) + adapter := &Adapter{ + kube: client, + resources: newMemoryResourceClient( + stableMachineConfigPool("master", 0), + stableMachineConfigPool("worker", 0), + stableMachineConfigPool("infra", 1), + ), + } + + _, err := adapter.PrepareReboot(ctx, guard, "replacement-key-id") + if err == nil || !strings.Contains(err.Error(), "matches 0 supported reboot targets") { + t.Fatalf("PrepareReboot() error = %v, want unsupported custom-pool rejection", err) + } +} + +func TestPrepareRebootUsesEffectiveMasterPoolForSingleNodeCluster(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + node := readyNode("master-0", "boot-before") + node.Labels["node-role.kubernetes.io/worker"] = "" + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + node, + ) + adapter := &Adapter{ + kube: client, + resources: newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 0), + ), + } + + plan, err := adapter.PrepareReboot(ctx, guard, "replacement-key-id") + if err != nil { + t.Fatalf("PrepareReboot() returned unexpected error: %v", err) + } + if !reflect.DeepEqual(plan.Targets, []string{"master"}) || len(plan.Baselines) != 1 || plan.Baselines[0].Target != "master" { + t.Fatalf("PrepareReboot() = %#v, want only the effective master target", plan) + } +} + +func TestPrepareRebootRejectsArbiterPoolBeforeMutation(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + node := readyNode("arbiter-0", "boot-before") + node.Labels = map[string]string{"node-role.kubernetes.io/arbiter": ""} + node.Annotations[nodeCurrentConfigAnnotation] = "rendered-arbiter-old" + node.Annotations[nodeDesiredConfigAnnotation] = "rendered-arbiter-old" + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + node, + ) + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 0), + stableMachineConfigPool("worker", 0), + stableMachineConfigPool("arbiter", 1), + ) + adapter := &Adapter{kube: client, resources: resources} + + _, err := adapter.PrepareReboot(ctx, guard, "replacement-key-id") + if err == nil || !strings.Contains(err.Error(), "exactly one of master or worker is required") { + t.Fatalf("PrepareReboot() error = %v, want arbiter topology rejection", err) + } + for _, target := range []string{"master", "worker"} { + creates, updates := resources.actionCount(machineConfigGVR, rebootMachineConfigName(target)) + if creates != 0 || updates != 0 { + t.Fatalf("%s MachineConfig mutated during failed planning: create/update=%d/%d", target, creates, updates) + } + } +} + +func TestPrepareRebootRejectsReservedMachineConfigCollisionBeforeMutation(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + readyNode("master-0", "boot-before"), + ) + collision := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "machineconfiguration.openshift.io/v1", + "kind": "MachineConfig", + "metadata": map[string]any{"name": rebootMachineConfigMaster}, + "spec": map[string]any{}, + }} + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 0), + collision, + ) + adapter := &Adapter{kube: client, resources: resources} + + _, err := adapter.PrepareReboot(ctx, guard, "replacement-key-id") + if err == nil || !strings.Contains(err.Error(), "without expected rotation ownership") { + t.Fatalf("PrepareReboot() error = %v, want reserved-name collision rejection", err) + } + creates, updates := resources.actionCount(machineConfigGVR, rebootMachineConfigMaster) + if creates != 0 || updates != 0 { + t.Fatalf("reserved MachineConfig mutated during failed planning: create/update=%d/%d", creates, updates) + } +} + +func TestWaitUntilReturnsPromptlyWhenContextIsCancelled(t *testing.T) { + adapter := &Adapter{options: Options{PollInterval: time.Hour}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + started := time.Now() + err := adapter.waitUntil(ctx, func(context.Context) (bool, error) { return false, nil }) + if err != context.Canceled { + t.Fatalf("waitUntil() error = %v, want context.Canceled", err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("waitUntil() took %s after cancellation", elapsed) + } +} + +func TestRequestRebootRecordsIntentBeforeMachineConfigsAndResumesMissingTarget(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + intent := testRebootIntent() + client := fake.NewSimpleClientset(testObjectsForRebootIntent(guard, intent)...) + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 1), + ) + workerKey := machineConfigGVR.String() + "/" + rebootMachineConfigWorker + resources.failCreateOnce[workerKey] = apierrors.NewServiceUnavailable("worker create interrupted") + resources.beforeCreate = func(resource schema.GroupVersionResource, _ *unstructured.Unstructured) error { + if resource != machineConfigGVR { + return nil + } + _, err := client.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace).Get(ctx, rebootRecordName(intent.ID), metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("MachineConfig mutation ran before canonical record: %w", err) + } + return nil + } + adapter := &Adapter{kube: client, resources: resources} + + outcome, err := adapter.RequestReboot(ctx, guard, intent) + if outcome != rotation.EffectUnknown || err == nil { + t.Fatalf("first RequestReboot() = %q, %v; want unknown partial outcome", outcome, err) + } + record, err := client.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace).Get(ctx, rebootRecordName(intent.ID), metav1.GetOptions{}) + if err != nil { + t.Fatalf("canonical reboot record was not durable: %v", err) + } + if record.Immutable == nil || !*record.Immutable { + t.Fatalf("canonical reboot record immutable = %#v", record.Immutable) + } + if _, err := resources.Get(ctx, machineConfigGVR, rebootMachineConfigMaster, metav1.GetOptions{}); err != nil { + t.Fatalf("master reboot request was not durable: %v", err) + } + if _, err := resources.Get(ctx, machineConfigGVR, rebootMachineConfigWorker, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("worker reboot request error = %v, want NotFound after interrupted create", err) + } + + outcome, err = adapter.RequestReboot(ctx, guard, intent) + if err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("resumed RequestReboot() = %q, %v", outcome, err) + } + outcome, err = adapter.RequestReboot(ctx, guard, intent) + if err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("idempotent RequestReboot() = %q, %v", outcome, err) + } + masterCreates, masterUpdates := resources.actionCount(machineConfigGVR, rebootMachineConfigMaster) + workerCreates, workerUpdates := resources.actionCount(machineConfigGVR, rebootMachineConfigWorker) + if masterCreates != 1 || masterUpdates != 0 || workerCreates != 2 || workerUpdates != 0 { + t.Fatalf("MachineConfig mutations: master create/update=%d/%d worker=%d/%d", masterCreates, masterUpdates, workerCreates, workerUpdates) + } +} + +func TestRequestRebootPreservesAmbiguousRecordCreateOutcome(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + intent := testSingleTargetRebootIntent() + client := fake.NewSimpleClientset(testObjectsForRebootIntent(guard, intent)...) + connectionLost := errors.New("connection lost after request submission") + client.Fake.PrependReactor("create", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*corev1.ConfigMap) + if created.Name == rebootRecordName(intent.ID) { + return true, nil, connectionLost + } + return false, nil, nil + }) + adapter := &Adapter{kube: client, resources: newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 0), + )} + + outcome, err := adapter.RequestReboot(ctx, guard, intent) + if outcome != rotation.EffectUnknown || !errors.Is(err, connectionLost) { + t.Fatalf("RequestReboot() = %q, %v; want EffectUnknown with connection loss", outcome, err) + } +} + +func TestRequestRebootRevalidatesPlanBeforeRecordingCanonicalIntent(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + intent := testSingleTargetRebootIntent() + node := readyNode(intent.Baselines[0].Node, "boot-changed-after-checkpoint") + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + node, + ) + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 0), + ) + adapter := &Adapter{kube: client, resources: resources} + + outcome, err := adapter.RequestReboot(ctx, guard, intent) + if outcome != rotation.EffectNotApplied || err == nil || !strings.Contains(err.Error(), "differs from the checkpointed intent") { + t.Fatalf("RequestReboot() = %q, %v; want stale-plan rejection", outcome, err) + } + if _, err := client.CoreV1().ConfigMaps(cloudCredentialOperatorNamespace).Get(ctx, rebootRecordName(intent.ID), metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("canonical reboot record error = %v, want NotFound", err) + } + creates, updates := resources.actionCount(machineConfigGVR, rebootMachineConfigMaster) + if creates != 0 || updates != 0 { + t.Fatalf("MachineConfig mutated for stale plan: create/update=%d/%d", creates, updates) + } +} + +func TestRequestRebootFailsClosedOnSameOperationMachineConfigDrift(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*unstructured.Unstructured) + }{ + { + name: "tampered marker", + mutate: func(machineConfig *unstructured.Unstructured) { + _ = unstructured.SetNestedSlice(machineConfig.Object, []any{map[string]any{ + "path": rebootMarkerPath, "mode": int64(0644), "overwrite": true, + "contents": map[string]any{"source": "data:,tampered"}, + }}, "spec", "config", "storage", "files") + }, + }, + { + name: "extra file", + mutate: func(machineConfig *unstructured.Unstructured) { + files, _, _ := unstructured.NestedSlice(machineConfig.Object, "spec", "config", "storage", "files") + files = append(files, map[string]any{ + "path": "/etc/unexpected", "mode": int64(0644), "overwrite": true, + "contents": map[string]any{"source": "data:,unexpected"}, + }) + _ = unstructured.SetNestedSlice(machineConfig.Object, files, "spec", "config", "storage", "files") + }, + }, + { + name: "extra spec field", + mutate: func(machineConfig *unstructured.Unstructured) { + _ = unstructured.SetNestedStringSlice(machineConfig.Object, []string{"debug"}, "spec", "kernelArguments") + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + intent := testSingleTargetRebootIntent() + client := fake.NewSimpleClientset(testObjectsForRebootIntent(guard, intent)...) + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 0), + ) + adapter := &Adapter{kube: client, resources: resources} + if outcome, err := adapter.RequestReboot(ctx, guard, intent); err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("initial RequestReboot() = %q, %v", outcome, err) + } + machineConfig, err := resources.Get(ctx, machineConfigGVR, rebootMachineConfigMaster, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + test.mutate(machineConfig) + resources.put(machineConfigGVR, machineConfig) + + outcome, err := adapter.RequestReboot(ctx, guard, intent) + if outcome != rotation.EffectUnknown || err == nil || !strings.Contains(err.Error(), "exact reboot specification") { + t.Fatalf("drifted RequestReboot() = %q, %v", outcome, err) + } + _, updates := resources.actionCount(machineConfigGVR, rebootMachineConfigMaster) + if updates != 0 { + t.Fatalf("drifted MachineConfig was updated %d times; want fail closed", updates) + } + }) + } +} + +func TestObserveRebootRequiresBootIDChangeAndAppliedRenderedMarker(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + intent := testSingleTargetRebootIntent() + node := readyNode("master-0", intent.Baselines[0].BootID) + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + node, + ) + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 0), + &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "machineconfiguration.openshift.io/v1", + "kind": "MachineConfig", + "metadata": map[string]any{"name": "rendered-master-new"}, + "spec": map[string]any{"config": map[string]any{}}, + }}, + ) + adapter := &Adapter{kube: client, resources: resources} + if outcome, err := adapter.RequestReboot(ctx, guard, intent); err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("RequestReboot() = %q, %v", outcome, err) + } + + observation, err := adapter.ObserveReboot(ctx, guard, intent.ID) + if err != nil || observation.Status != rotation.RebootInProgress { + t.Fatalf("observation before reboot = %#v, %v", observation, err) + } + current, err := client.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + current.Status.NodeInfo.BootID = "master-boot-after" + current.Annotations[nodeCurrentConfigAnnotation] = "rendered-master-new" + current.Annotations[nodeDesiredConfigAnnotation] = "rendered-master-new" + if _, err := client.CoreV1().Nodes().UpdateStatus(ctx, current, metav1.UpdateOptions{}); err != nil { + t.Fatalf("update node boot ID: %v", err) + } + observation, err = adapter.ObserveReboot(ctx, guard, intent.ID) + if err != nil || observation.Status != rotation.RebootInProgress { + t.Fatalf("observation without rendered marker = %#v, %v", observation, err) + } + selectorLabels, err := machineConfigSelectorLabels(stableMachineConfigPool("master", 1)) + if err != nil { + t.Fatal(err) + } + resources.put(machineConfigGVR, desiredRebootMachineConfig(nil, "rendered-master-new", "master", intent.ID, selectorLabels)) + observation, err = adapter.ObserveReboot(ctx, guard, intent.ID) + if err != nil || observation.Status != rotation.RebootComplete { + t.Fatalf("completed observation = %#v, %v", observation, err) + } + if observation.CanonicalIntent == nil || !reflect.DeepEqual(*observation.CanonicalIntent, intent) { + t.Fatalf("canonical intent = %#v, want %#v", observation.CanonicalIntent, intent) + } +} + +func TestObserveRebootTreatsDeletedBaselineNodeAsComplete(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + intent := testSingleTargetRebootIntent() + originalNode := readyNode("master-0", intent.Baselines[0].BootID) + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + originalNode, + ) + resources := newMemoryResourceClient( + stableMachineConfigPool("master", 1), + stableMachineConfigPool("worker", 0), + ) + adapter := newAdapter( + client, + staticClusterVersionGetter{}, + staticClusterOperatorLister{operators: stableClusterOperators()}, + resources, + staticSecretMetadataGetter{}, + testOptions(), + ) + if outcome, err := adapter.RequestReboot(ctx, guard, intent); err != nil || outcome != rotation.EffectSubmitted { + t.Fatalf("RequestReboot() = %q, %v", outcome, err) + } + if err := client.CoreV1().Nodes().Delete(ctx, originalNode.Name, metav1.DeleteOptions{}); err != nil { + t.Fatalf("delete baseline node: %v", err) + } + replacementNode := readyNode("master-replacement", "replacement-boot") + replacementNode.Annotations[nodeCurrentConfigAnnotation] = "rendered-master-replacement" + replacementNode.Annotations[nodeDesiredConfigAnnotation] = "rendered-master-replacement" + if _, err := client.CoreV1().Nodes().Create(ctx, replacementNode, metav1.CreateOptions{}); err != nil { + t.Fatalf("create replacement node: %v", err) + } + masterPool := stableMachineConfigPool("master", 1) + _ = unstructured.SetNestedField(masterPool.Object, "rendered-master-replacement", "spec", "configuration", "name") + _ = unstructured.SetNestedField(masterPool.Object, "rendered-master-replacement", "status", "configuration", "name") + resources.put(machineConfigPoolGVR, masterPool) + selectorLabels, err := machineConfigSelectorLabels(stableMachineConfigPool("master", 1)) + if err != nil { + t.Fatal(err) + } + resources.put(machineConfigGVR, desiredRebootMachineConfig(nil, "rendered-master-replacement", "master", intent.ID, selectorLabels)) + + observation, err := adapter.ObserveReboot(ctx, guard, intent.ID) + if err != nil || observation.Status != rotation.RebootComplete { + t.Fatalf("ObserveReboot() after baseline node deletion = %#v, %v", observation, err) + } + if err := adapter.WaitForPostRebootStable(ctx, guard, intent); err != nil { + t.Fatalf("WaitForPostRebootStable() after baseline node replacement: %v", err) + } +} + +func TestWaitForSignerRolloutRequiresEveryCurrentRevision(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + publicKey := testPublicKeyPEM(t) + set, err := jwks.NewSigner(publicKey) + if err != nil { + t.Fatalf("build test signer: %v", err) + } + entryName := "service-account-002.pub" + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: kubeAPIServerNamespace, Name: publicSignerCMName}, Data: map[string]string{entryName: string(publicKey)}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: kubeAPIServerNamespace, Name: publicSignerCMName + "-7"}, Data: map[string]string{entryName: string(publicKey)}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: kubeAPIServerNamespace, Name: publicSignerCMName + "-8"}, Data: map[string]string{entryName: string(publicKey)}}, + ) + adapter := newAdapter( + client, + staticClusterVersionGetter{}, + staticClusterOperatorLister{operators: stableClusterOperators()}, + newMemoryResourceClient(testKubeAPIServer(7, 8)), + staticSecretMetadataGetter{}, + testOptions(), + ) + if err := adapter.WaitForSignerRollout(ctx, guard, set.Keys[0].KeyID); err != nil { + t.Fatalf("WaitForSignerRollout() returned unexpected error: %v", err) + } +} + +func TestReconcileSignerRolloutCheckpointRequiresCurrentStability(t *testing.T) { + ctx := context.Background() + guard := testGuardReference() + clusterID := configv1.ClusterID("a58a4337-2d99-41ed-b009-33b62d854224") + client := fake.NewSimpleClientset( + newGuardConfigMap(guardState{SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID}), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: kubeAPIServerNamespace, Name: publicSignerCMName}, + Data: map[string]string{"service-account-001.pub": string(testPublicKeyPEM(t))}, + }, + ) + adapter := newAdapter( + client, + staticClusterVersionGetter{version: &configv1.ClusterVersion{Spec: configv1.ClusterVersionSpec{ClusterID: clusterID}}}, + staticClusterOperatorLister{operators: stableClusterOperators()}, + newMemoryResourceClient(), + staticSecretMetadataGetter{}, + testOptions(), + ) + expectation := rotation.ClusterExpectation{ + Phase: rotation.PhaseSignerRolloutStable, + ClusterIdentity: string(clusterID), + RotationGuard: guard, + ReplacementSigner: &rotation.ReplacementSignerEvidence{ + Entry: rotation.PublicSignerBaselineEntry{KeyID: "replacement-key-not-published"}, + }, + } + + err := adapter.Reconcile(ctx, guard, expectation) + if err == nil || !strings.Contains(err.Error(), "cluster is not currently stable") { + t.Fatalf("Reconcile() error = %v, want stale signer-rollout checkpoint rejection", err) + } +} + +func testGuardReference() rotation.RotationGuardReference { + return rotation.RotationGuardReference{ScopeID: strings.Repeat("a", 64), OperationID: strings.Repeat("b", 64)} +} + +func testRebootIntent() rotation.RebootIntent { + return rotation.RebootIntent{ + ID: "signer-rotation-" + strings.Repeat("c", 64), + Targets: []string{"master", "worker"}, + Baselines: []rotation.NodeRebootBaseline{ + {Target: "master", Node: "master-0", BootID: "master-boot-before"}, + {Target: "worker", Node: "worker-0", BootID: "worker-boot-before"}, + }, + } +} + +func testSingleTargetRebootIntent() rotation.RebootIntent { + return rotation.RebootIntent{ + ID: "signer-rotation-" + strings.Repeat("d", 64), + Targets: []string{"master"}, + Baselines: []rotation.NodeRebootBaseline{{Target: "master", Node: "master-0", BootID: "master-boot-before"}}, + } +} + +func testObjectsForRebootIntent(guard rotation.RotationGuardReference, intent rotation.RebootIntent) []runtime.Object { + objects := []runtime.Object{newGuardConfigMap(guardState{ + SchemaVersion: guardSchemaVersion, ScopeID: guard.ScopeID, ActiveOperationID: guard.OperationID, + })} + for _, baseline := range intent.Baselines { + node := readyNode(baseline.Node, baseline.BootID) + node.Labels = map[string]string{"node-role.kubernetes.io/" + baseline.Target: ""} + node.Annotations[nodeCurrentConfigAnnotation] = "rendered-" + baseline.Target + "-old" + node.Annotations[nodeDesiredConfigAnnotation] = "rendered-" + baseline.Target + "-old" + objects = append(objects, node) + } + return objects +} + +func readyNode(name, bootID string) *corev1.Node { + role := testNodeRole(name) + return &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "node-role.kubernetes.io/" + role: "", + }, + Annotations: map[string]string{ + nodeCurrentConfigAnnotation: "rendered-" + role + "-old", + nodeDesiredConfigAnnotation: "rendered-" + role + "-old", + }, + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{BootID: bootID}, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + }, + } +} + +func testPublicKeyPEM(t *testing.T) []byte { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate RSA key: %v", err) + } + der, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey) + if err != nil { + t.Fatalf("marshal RSA public key: %v", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der}) +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/signer.go b/pkg/cmd/provisioning/rotation/kubeadapter/signer.go new file mode 100644 index 0000000000..4cf20bd227 --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/signer.go @@ -0,0 +1,198 @@ +package kubeadapter + +import ( + "bytes" + "context" + "fmt" + "slices" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/jwks" + "github.com/openshift/cloud-credential-operator/pkg/cmd/provisioning/rotation" +) + +func (a *Adapter) ObserveSignerReference(ctx context.Context, guard *rotation.RotationGuardReference) (*rotation.SignerObjectReference, error) { + if guard != nil { + if err := a.requireGuardHeld(ctx, *guard); err != nil { + return nil, err + } + } + metadata, err := a.secretMetadata.Get(ctx, kubeAPIServerOperatorNamespace, nextSignerSecretName) + if apierrors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read next signer Secret metadata: %w", err) + } + if metadata.UID == "" || metadata.ResourceVersion == "" { + return nil, fmt.Errorf("next signer Secret metadata is missing UID or resourceVersion") + } + return &rotation.SignerObjectReference{ + UID: string(metadata.UID), + ResourceVersion: metadata.ResourceVersion, + }, nil +} + +func (a *Adapter) ObservePublicSignerBundle(ctx context.Context, guard *rotation.RotationGuardReference) (rotation.PublicSignerBundleObservation, error) { + if guard != nil { + if err := a.requireGuardHeld(ctx, *guard); err != nil { + return rotation.PublicSignerBundleObservation{}, err + } + } + configMap, err := a.kube.CoreV1().ConfigMaps(kubeAPIServerNamespace).Get(ctx, publicSignerCMName, metav1.GetOptions{}) + if err != nil { + return rotation.PublicSignerBundleObservation{}, fmt.Errorf("read public signer ConfigMap: %w", err) + } + if configMap.UID == "" || configMap.ResourceVersion == "" { + return rotation.PublicSignerBundleObservation{}, fmt.Errorf("public signer ConfigMap is missing UID or resourceVersion") + } + if len(configMap.BinaryData) != 0 { + return rotation.PublicSignerBundleObservation{}, fmt.Errorf("public signer ConfigMap contains unsupported binary data") + } + names := make([]string, 0, len(configMap.Data)) + for name := range configMap.Data { + names = append(names, name) + } + slices.Sort(names) + signers := make([]rotation.PublicSignerObservation, 0, len(names)) + for _, name := range names { + signers = append(signers, rotation.PublicSignerObservation{ + Name: name, + PublicKeyPEM: []byte(configMap.Data[name]), + }) + } + return rotation.PublicSignerBundleObservation{ + ConfigMapUID: string(configMap.UID), + ConfigMapResourceVersion: configMap.ResourceVersion, + Signers: signers, + }, nil +} + +func (a *Adapter) RequestReplacement(ctx context.Context, guard rotation.RotationGuardReference, reference rotation.SignerObjectReference) (rotation.EffectOutcome, error) { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return rotation.EffectNotApplied, err + } + if strings.TrimSpace(reference.UID) == "" || strings.TrimSpace(reference.ResourceVersion) == "" { + return rotation.EffectNotApplied, fmt.Errorf("signer Secret deletion requires UID and resourceVersion preconditions") + } + uid := types.UID(reference.UID) + resourceVersion := reference.ResourceVersion + err := a.kube.CoreV1().Secrets(kubeAPIServerOperatorNamespace).Delete(ctx, nextSignerSecretName, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{ + UID: &uid, + ResourceVersion: &resourceVersion, + }, + }) + return mutationOutcome(err), err +} + +func (a *Adapter) WaitForReplacement(ctx context.Context, guard rotation.RotationGuardReference, previous rotation.SignerObjectReference) error { + if strings.TrimSpace(previous.UID) == "" { + return fmt.Errorf("previous signer Secret UID must not be empty") + } + return a.waitUntil(ctx, func(ctx context.Context) (bool, error) { + reference, err := a.ObserveSignerReference(ctx, &guard) + if err != nil { + return false, err + } + if reference == nil || reference.UID == previous.UID { + return false, nil + } + bundle, err := a.ObservePublicSignerBundle(ctx, &guard) + if err != nil { + return false, err + } + return len(bundle.Signers) != 0, nil + }) +} + +func (a *Adapter) WaitForSignerRollout(ctx context.Context, guard rotation.RotationGuardReference, replacementKeyID string) error { + if strings.TrimSpace(replacementKeyID) == "" || strings.TrimSpace(replacementKeyID) != replacementKeyID { + return fmt.Errorf("replacement signer key ID must not be empty or contain surrounding whitespace") + } + return a.waitForContinuousStability(ctx, a.options.StablePeriod, func(ctx context.Context) (bool, error) { + if err := a.requireGuardHeld(ctx, guard); err != nil { + return false, err + } + return a.signerRolloutStable(ctx, replacementKeyID) + }) +} + +func (a *Adapter) signerRolloutStable(ctx context.Context, replacementKeyID string) (bool, error) { + entryName, publicKey, found, err := a.findPublicSignerByKeyID(ctx, replacementKeyID) + if err != nil || !found { + return false, err + } + distributed, err := a.signerDistributedToCurrentKubeAPIServerRevisions(ctx, entryName, publicKey) + if err != nil || !distributed { + return false, err + } + return a.clusterOperatorsStable(ctx) +} + +func (a *Adapter) findPublicSignerByKeyID(ctx context.Context, keyID string) (string, []byte, bool, error) { + configMap, err := a.kube.CoreV1().ConfigMaps(kubeAPIServerNamespace).Get(ctx, publicSignerCMName, metav1.GetOptions{}) + if err != nil { + return "", nil, false, fmt.Errorf("read public signer ConfigMap during rollout: %w", err) + } + if len(configMap.BinaryData) != 0 { + return "", nil, false, fmt.Errorf("public signer ConfigMap contains unsupported binary data") + } + for name, value := range configMap.Data { + set, err := jwks.NewSigner([]byte(value)) + if err != nil { + return "", nil, false, fmt.Errorf("parse public signer ConfigMap entry %q: %w", name, err) + } + if len(set.Keys) != 1 { + return "", nil, false, fmt.Errorf("public signer ConfigMap entry %q did not produce exactly one key", name) + } + if set.Keys[0].KeyID == keyID { + return name, []byte(value), true, nil + } + } + return "", nil, false, nil +} + +func (a *Adapter) signerDistributedToCurrentKubeAPIServerRevisions(ctx context.Context, entryName string, publicKey []byte) (bool, error) { + kubeAPIServer, err := a.resources.Get(ctx, kubeAPIServerGVR, "cluster", metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("read kube-apiserver operator status: %w", err) + } + nodeStatuses, found, err := unstructuredSlice(kubeAPIServer, "status", "nodeStatuses") + if err != nil { + return false, err + } + if !found || len(nodeStatuses) == 0 { + return false, nil + } + revisions := make(map[int64]struct{}) + for _, item := range nodeStatuses { + status, ok := item.(map[string]any) + if !ok { + return false, fmt.Errorf("kube-apiserver node status has type %T", item) + } + revision, ok := integerValue(status["currentRevision"]) + if !ok || revision <= 0 { + return false, nil + } + revisions[revision] = struct{}{} + } + for revision := range revisions { + name := fmt.Sprintf("%s-%d", publicSignerCMName, revision) + configMap, err := a.kube.CoreV1().ConfigMaps(kubeAPIServerNamespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("read kube-apiserver signer revision ConfigMap %q: %w", name, err) + } + if len(configMap.BinaryData) != 0 || !bytes.Equal([]byte(configMap.Data[entryName]), publicKey) { + return false, nil + } + } + return true, nil +} diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/test_helpers_test.go b/pkg/cmd/provisioning/rotation/kubeadapter/test_helpers_test.go new file mode 100644 index 0000000000..dbc4a9b51b --- /dev/null +++ b/pkg/cmd/provisioning/rotation/kubeadapter/test_helpers_test.go @@ -0,0 +1,253 @@ +package kubeadapter + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + "time" + + configv1 "github.com/openshift/api/config/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type staticClusterVersionGetter struct { + version *configv1.ClusterVersion + err error +} + +func (s staticClusterVersionGetter) Get(context.Context, string, metav1.GetOptions) (*configv1.ClusterVersion, error) { + if s.err != nil { + return nil, s.err + } + return s.version.DeepCopy(), nil +} + +type staticClusterOperatorLister struct { + operators *configv1.ClusterOperatorList + err error +} + +func (s staticClusterOperatorLister) List(context.Context, metav1.ListOptions) (*configv1.ClusterOperatorList, error) { + if s.err != nil { + return nil, s.err + } + return s.operators.DeepCopy(), nil +} + +type staticSecretMetadataGetter struct { + metadata *metav1.PartialObjectMetadata + err error +} + +func (s staticSecretMetadataGetter) Get(context.Context, string, string) (*metav1.PartialObjectMetadata, error) { + if s.err != nil { + return nil, s.err + } + return s.metadata.DeepCopy(), nil +} + +type memoryResourceClient struct { + mu sync.Mutex + + objects map[schema.GroupVersionResource]map[string]*unstructured.Unstructured + failCreateOnce map[string]error + createCalls map[string]int + updateCalls map[string]int + beforeCreate func(schema.GroupVersionResource, *unstructured.Unstructured) error + nextResourceVersion int +} + +func newMemoryResourceClient(objects ...*unstructured.Unstructured) *memoryResourceClient { + client := &memoryResourceClient{ + objects: map[schema.GroupVersionResource]map[string]*unstructured.Unstructured{}, + failCreateOnce: map[string]error{}, + createCalls: map[string]int{}, + updateCalls: map[string]int{}, + nextResourceVersion: 1, + } + for _, object := range objects { + client.putLocked(gvrForObject(object), object) + } + return client +} + +func (c *memoryResourceClient) Get(_ context.Context, resource schema.GroupVersionResource, name string, _ metav1.GetOptions) (*unstructured.Unstructured, error) { + c.mu.Lock() + defer c.mu.Unlock() + object := c.objects[resource][name] + if object == nil { + return nil, apierrors.NewNotFound(resource.GroupResource(), name) + } + return object.DeepCopy(), nil +} + +func (c *memoryResourceClient) List(_ context.Context, resource schema.GroupVersionResource, _ metav1.ListOptions) (*unstructured.UnstructuredList, error) { + c.mu.Lock() + defer c.mu.Unlock() + list := &unstructured.UnstructuredList{} + names := make([]string, 0, len(c.objects[resource])) + for name := range c.objects[resource] { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + list.Items = append(list.Items, *c.objects[resource][name].DeepCopy()) + } + return list, nil +} + +func (c *memoryResourceClient) Create(_ context.Context, resource schema.GroupVersionResource, object *unstructured.Unstructured, _ metav1.CreateOptions) (*unstructured.Unstructured, error) { + c.mu.Lock() + defer c.mu.Unlock() + key := resource.String() + "/" + object.GetName() + c.createCalls[key]++ + if c.beforeCreate != nil { + if err := c.beforeCreate(resource, object.DeepCopy()); err != nil { + return nil, err + } + } + if err := c.failCreateOnce[key]; err != nil { + delete(c.failCreateOnce, key) + return nil, err + } + if c.objects[resource][object.GetName()] != nil { + return nil, apierrors.NewAlreadyExists(resource.GroupResource(), object.GetName()) + } + created := object.DeepCopy() + created.SetResourceVersion(fmt.Sprintf("%d", c.nextResourceVersion)) + c.nextResourceVersion++ + c.putLocked(resource, created) + return created.DeepCopy(), nil +} + +func (c *memoryResourceClient) Update(_ context.Context, resource schema.GroupVersionResource, object *unstructured.Unstructured, _ metav1.UpdateOptions) (*unstructured.Unstructured, error) { + c.mu.Lock() + defer c.mu.Unlock() + key := resource.String() + "/" + object.GetName() + c.updateCalls[key]++ + current := c.objects[resource][object.GetName()] + if current == nil { + return nil, apierrors.NewNotFound(resource.GroupResource(), object.GetName()) + } + if object.GetResourceVersion() != current.GetResourceVersion() { + return nil, apierrors.NewConflict(resource.GroupResource(), object.GetName(), fmt.Errorf("resourceVersion changed")) + } + updated := object.DeepCopy() + updated.SetResourceVersion(fmt.Sprintf("%d", c.nextResourceVersion)) + c.nextResourceVersion++ + c.putLocked(resource, updated) + return updated.DeepCopy(), nil +} + +func (c *memoryResourceClient) put(resource schema.GroupVersionResource, object *unstructured.Unstructured) { + c.mu.Lock() + defer c.mu.Unlock() + c.putLocked(resource, object) +} + +func (c *memoryResourceClient) putLocked(resource schema.GroupVersionResource, object *unstructured.Unstructured) { + if c.objects[resource] == nil { + c.objects[resource] = map[string]*unstructured.Unstructured{} + } + copy := object.DeepCopy() + if copy.GetResourceVersion() == "" { + copy.SetResourceVersion(fmt.Sprintf("%d", c.nextResourceVersion)) + c.nextResourceVersion++ + } + c.objects[resource][copy.GetName()] = copy +} + +func (c *memoryResourceClient) actionCount(resource schema.GroupVersionResource, name string) (creates, updates int) { + c.mu.Lock() + defer c.mu.Unlock() + key := resource.String() + "/" + name + return c.createCalls[key], c.updateCalls[key] +} + +func gvrForObject(object *unstructured.Unstructured) schema.GroupVersionResource { + switch object.GetKind() { + case "MachineConfigPool": + return machineConfigPoolGVR + case "MachineConfig": + return machineConfigGVR + case "KubeAPIServer": + return kubeAPIServerGVR + default: + panic(fmt.Sprintf("unsupported test object kind %q", object.GetKind())) + } +} + +func testOptions() Options { + return Options{PollInterval: time.Millisecond, PreflightStablePeriod: 0, StablePeriod: 0} +} + +func stableClusterOperators() *configv1.ClusterOperatorList { + return &configv1.ClusterOperatorList{Items: []configv1.ClusterOperator{{ + ObjectMeta: metav1.ObjectMeta{Name: "kube-apiserver"}, + Status: configv1.ClusterOperatorStatus{Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorDegraded, Status: configv1.ConditionFalse}, + }}, + }}} +} + +func stableMachineConfigPool(name string, machineCount int64) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "machineconfiguration.openshift.io/v1", + "kind": "MachineConfigPool", + "metadata": map[string]any{ + "name": name, + "generation": int64(2), + }, + "spec": map[string]any{ + "paused": false, + "nodeSelector": map[string]any{ + "matchLabels": map[string]any{"node-role.kubernetes.io/" + name: ""}, + }, + "machineConfigSelector": map[string]any{ + "matchLabels": map[string]any{"machineconfiguration.openshift.io/role": name}, + }, + "configuration": map[string]any{"name": "rendered-" + name + "-old"}, + }, + "status": map[string]any{ + "observedGeneration": int64(2), + "configuration": map[string]any{"name": "rendered-" + name + "-old"}, + "machineCount": machineCount, + "updatedMachineCount": machineCount, + "readyMachineCount": machineCount, + "unavailableMachineCount": int64(0), + "degradedMachineCount": int64(0), + "conditions": []any{ + map[string]any{"type": "Updated", "status": "True"}, + map[string]any{"type": "Updating", "status": "False"}, + map[string]any{"type": "Degraded", "status": "False"}, + }, + }, + }} +} + +func testNodeRole(name string) string { + if strings.HasPrefix(name, "master") { + return "master" + } + return "worker" +} + +func testKubeAPIServer(revisions ...int64) *unstructured.Unstructured { + statuses := make([]any, 0, len(revisions)) + for index, revision := range revisions { + statuses = append(statuses, map[string]any{"nodeName": fmt.Sprintf("master-%d", index), "currentRevision": revision}) + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "operator.openshift.io/v1", + "kind": "KubeAPIServer", + "metadata": map[string]any{"name": "cluster"}, + "status": map[string]any{"nodeStatuses": statuses}, + }} +} diff --git a/pkg/cmd/provisioning/rotation/orchestrator.go b/pkg/cmd/provisioning/rotation/orchestrator.go index 9e613c0965..96768343cb 100644 --- a/pkg/cmd/provisioning/rotation/orchestrator.go +++ b/pkg/cmd/provisioning/rotation/orchestrator.go @@ -565,7 +565,7 @@ func (o Orchestrator) completeReboot(ctx context.Context, workspace *RotationWor return advanceCheckpoint(workspace, checkpoint, PhaseNodesRebooted, nil) } - if status == RebootNotStarted { + if status == RebootNotStarted || status == RebootInProgress { outcome, requestErr := o.Cluster.RequestReboot(ctx, *checkpoint.RotationGuard, cloneRebootIntent(intent)) if err := validateEffectOutcome(outcome); err != nil { return err @@ -580,15 +580,20 @@ func (o Orchestrator) completeReboot(ctx context.Context, workspace *RotationWor if status == RebootComplete { return advanceCheckpoint(workspace, checkpoint, PhaseNodesRebooted, nil) } - if status == RebootNotStarted { - switch outcome { - case EffectUnknown, EffectSubmitted: + switch outcome { + case EffectUnknown: + return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "reconcile persisted node reboot request", Cause: requestErr} + case EffectNotApplied: + if requestErr != nil { + return fmt.Errorf("node reboot request was not applied: %w", requestErr) + } + return &ConflictError{Phase: checkpoint.Phase, Reason: "the persisted reboot request was not applied"} + case EffectSubmitted: + if requestErr != nil { + return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "reconcile persisted node reboot request", Cause: requestErr} + } + if status == RebootNotStarted { return &OutcomeUnknownError{Phase: checkpoint.Phase, Operation: "request persisted node reboot", Cause: requestErr} - case EffectNotApplied: - if requestErr != nil { - return fmt.Errorf("node reboot request was not applied: %w", requestErr) - } - return &ConflictError{Phase: checkpoint.Phase, Reason: "the persisted reboot request was not applied"} } } } diff --git a/pkg/cmd/provisioning/rotation/orchestrator_safety_test.go b/pkg/cmd/provisioning/rotation/orchestrator_safety_test.go index c54e664ffa..1854f5e39d 100644 --- a/pkg/cmd/provisioning/rotation/orchestrator_safety_test.go +++ b/pkg/cmd/provisioning/rotation/orchestrator_safety_test.go @@ -72,7 +72,7 @@ func TestOrchestratorDoesNotAdvanceUntilSubmittedReplacementIsObserved(t *testin } } -func TestOrchestratorResumeDoesNotRepeatUnknownRebootRequest(t *testing.T) { +func TestOrchestratorResumeDoesNotReenterUnknownRebootAfterCompletionIsObserved(t *testing.T) { oldPublic := testPublicKeyPEM(t) newPublic := testPublicKeyPEM(t) oldSigner := safetySigner(oldPublic, "old-uid", "10") diff --git a/pkg/cmd/provisioning/rotation/orchestrator_test.go b/pkg/cmd/provisioning/rotation/orchestrator_test.go index 2732a2b4b6..d153d77d30 100644 --- a/pkg/cmd/provisioning/rotation/orchestrator_test.go +++ b/pkg/cmd/provisioning/rotation/orchestrator_test.go @@ -546,6 +546,7 @@ type fakeClusterRotation struct { waitForSignerError error replacementRequests int rebootRequests int + rebootMutations int bundleObservations int applyReplacementAt int requestedReference SignerObjectReference @@ -733,6 +734,7 @@ func (f *fakeClusterRotation) RequestReboot(_ context.Context, guard RotationGua if f.canonicalReboot == nil { canonical := cloneRebootIntent(intent) f.canonicalReboot = &canonical + f.rebootMutations++ } f.rebootStatus = RebootInProgress return EffectSubmitted, nil diff --git a/pkg/cmd/provisioning/rotation/orchestrator_types.go b/pkg/cmd/provisioning/rotation/orchestrator_types.go index 2903bf08e5..8db6b4a65a 100644 --- a/pkg/cmd/provisioning/rotation/orchestrator_types.go +++ b/pkg/cmd/provisioning/rotation/orchestrator_types.go @@ -72,6 +72,8 @@ type RotationGuardObservation struct { } // RebootStatus is the observable state of a cluster-durable reboot record. +// RebootInProgress includes the interval after the canonical intent is durable +// but before every target reboot request has been reconciled. type RebootStatus string const ( @@ -109,12 +111,23 @@ type RebootObservation struct { // may already be absent or replaced, and at PhaseRebootIntentRecorded the exact // persisted reboot may be not started, in progress, or complete. // -// RequestReboot is the sole reboot mutation. It must atomically create the -// cluster-durable canonical record and request the reboot when the ID is new. -// For an existing ID it must preserve the first canonical intent and must not -// request another reboot. The canonical record must remain observable while a +// RequestReboot is the sole reboot mutation. When the ID is new, it must first +// durably create the cluster-canonical intent and must not mutate a reboot +// target before that record exists. It must then idempotently reconcile every +// target request in the canonical intent. For an existing ID it must preserve +// the first canonical intent and finish any target requests that are not yet +// durable, without repeating a target request that already records the same +// operation ID. The orchestrator may therefore call RequestReboot again while +// the operation is RebootInProgress. +// +// EffectSubmitted means the canonical intent and every target request are +// known to be durable, including when they were already in the desired state. +// A partial or ambiguous multi-target result must be EffectUnknown; +// EffectNotApplied is valid only when the adapter knows that no mutation from +// the call was applied. The canonical record must remain observable while a // checkpoint for the operation can be resumed. RebootNotStarted means that no -// queued, active, or completed operation with that ID is observable. +// canonical record, queued target, active target, or completed operation with +// that ID is observable. // ObserveSignerReference must request only the meta.k8s.io/v1 // PartialObjectMetadata representation of the next-signer Secret and must fail // closed rather than accepting a full Secret fallback. A nil reference means diff --git a/pkg/cmd/provisioning/rotation/reboot_cancellation_test.go b/pkg/cmd/provisioning/rotation/reboot_cancellation_test.go index 72f0278a60..2f7867925e 100644 --- a/pkg/cmd/provisioning/rotation/reboot_cancellation_test.go +++ b/pkg/cmd/provisioning/rotation/reboot_cancellation_test.go @@ -120,7 +120,7 @@ func TestOrchestratorRebootEffectNotAppliedDoesNotAdvanceOrWait(t *testing.T) { } } -func TestOrchestratorRebootReadbackFailureResumesWithoutDuplicateRequest(t *testing.T) { +func TestOrchestratorRebootReadbackFailureReconcilesWithoutDuplicateMutation(t *testing.T) { harness := newOrchestratorTestHarness(t, PublicationModeDirect) cluster := &rebootCoverageCluster{ fakeClusterRotation: harness.cluster, @@ -154,8 +154,56 @@ func TestOrchestratorRebootReadbackFailureResumesWithoutDuplicateRequest(t *test if !result.Complete || result.Phase != PhaseComplete { t.Fatalf("resumed Run() result = %#v, want complete", result) } - if harness.cluster.rebootRequests != 1 { - t.Fatalf("reboot requests after resume = %d, want exactly one", harness.cluster.rebootRequests) + if harness.cluster.rebootRequests != 2 || harness.cluster.rebootMutations != 1 { + t.Fatalf("reboot request calls/mutations after resume = %d/%d, want 2/1", harness.cluster.rebootRequests, harness.cluster.rebootMutations) + } +} + +func TestOrchestratorPartialRebootRequestIsReconciledBeforeWait(t *testing.T) { + harness := newOrchestratorTestHarness(t, PublicationModeDirect) + harness.cluster.rebootIntent = RebootIntent{ + ID: "rotation-test-intent", + Targets: []string{"master", "worker"}, + Baselines: []NodeRebootBaseline{ + {Target: "master", Node: "master-0", BootID: "master-boot-old"}, + {Target: "worker", Node: "worker-0", BootID: "worker-boot-old"}, + }, + } + cluster := &partialRebootCluster{ + fakeClusterRotation: harness.cluster, + targetRequests: make(map[string]int), + } + harness.orchestrator.Cluster = cluster + outputDir := t.TempDir() + options := RunOptions{Provider: ProviderAWS, PublicationMode: PublicationModeDirect, OutputDir: outputDir} + + result, err := harness.orchestrator.Run(context.Background(), options) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("initial Run() error = %v, want OutcomeUnknownError", err) + } + if result.Phase != PhaseRebootIntentRecorded || cluster.requestCalls != 1 || cluster.canonicalWrites != 1 || cluster.waitCalls != 0 { + t.Fatalf("initial phase/request calls/canonical writes/waits = %q/%d/%d/%d, want %q/1/1/0", result.Phase, cluster.requestCalls, cluster.canonicalWrites, cluster.waitCalls, PhaseRebootIntentRecorded) + } + if cluster.targetRequests["master"] != 1 || cluster.targetRequests["worker"] != 0 { + t.Fatalf("initial target requests = %v, want only master requested", cluster.targetRequests) + } + + options.Resume = true + result, err = harness.orchestrator.Run(context.Background(), options) + if err != nil { + t.Fatalf("resumed Run() returned unexpected error: %v", err) + } + if !result.Complete || result.Phase != PhaseComplete { + t.Fatalf("resumed Run() result = %#v, want complete", result) + } + if cluster.requestCalls != 2 || cluster.canonicalWrites != 1 || cluster.waitCalls != 1 { + t.Fatalf("final request calls/canonical writes/waits = %d/%d/%d, want 2/1/1", cluster.requestCalls, cluster.canonicalWrites, cluster.waitCalls) + } + for _, target := range []string{"master", "worker"} { + if cluster.targetRequests[target] != 1 { + t.Fatalf("target %q request mutations = %d, want 1; all targets = %v", target, cluster.targetRequests[target], cluster.targetRequests) + } } } @@ -190,8 +238,8 @@ func TestOrchestratorRejectsWaitThatReturnsBeforeCanonicalRebootCompletion(t *te if !errors.As(err, &conflict) { t.Fatalf("resumed Run() error = %v, want ConflictError", err) } - if result.Phase != PhaseRebootIntentRecorded || harness.cluster.rebootRequests != 1 || cluster.waitCalls != 2 { - t.Fatalf("resumed phase/requests/waits = %q/%d/%d, want %q/1/2", result.Phase, harness.cluster.rebootRequests, cluster.waitCalls, PhaseRebootIntentRecorded) + if result.Phase != PhaseRebootIntentRecorded || harness.cluster.rebootRequests != 2 || harness.cluster.rebootMutations != 1 || cluster.waitCalls != 2 { + t.Fatalf("resumed phase/request calls/mutations/waits = %q/%d/%d/%d, want %q/2/1/2", result.Phase, harness.cluster.rebootRequests, harness.cluster.rebootMutations, cluster.waitCalls, PhaseRebootIntentRecorded) } } @@ -208,6 +256,43 @@ type rebootCoverageCluster struct { waitCalls int } +type partialRebootCluster struct { + *fakeClusterRotation + requestCalls int + canonicalWrites int + targetRequests map[string]int + waitCalls int +} + +func (cluster *partialRebootCluster) RequestReboot(_ context.Context, guard RotationGuardReference, intent RebootIntent) (EffectOutcome, error) { + *cluster.events = append(*cluster.events, "cluster.request-reboot") + if err := cluster.requireHeldRotationGuard(guard); err != nil { + return EffectNotApplied, err + } + cluster.requestCalls++ + if cluster.canonicalReboot == nil { + canonical := cloneRebootIntent(intent) + cluster.canonicalReboot = &canonical + cluster.canonicalWrites++ + } + cluster.rebootStatus = RebootInProgress + for _, target := range cluster.canonicalReboot.Targets { + if cluster.targetRequests[target] != 0 { + continue + } + cluster.targetRequests[target]++ + if cluster.requestCalls == 1 { + return EffectUnknown, errors.New("remaining reboot target request outcome is unknown") + } + } + return EffectSubmitted, nil +} + +func (cluster *partialRebootCluster) WaitForReboot(ctx context.Context, guard RotationGuardReference, intent RebootIntent) error { + cluster.waitCalls++ + return cluster.fakeClusterRotation.WaitForReboot(ctx, guard, intent) +} + func (cluster *rebootCoverageCluster) PrepareReboot(ctx context.Context, guard RotationGuardReference, replacementKeyID string) (RebootPlan, error) { plan, err := cluster.fakeClusterRotation.PrepareReboot(ctx, guard, replacementKeyID) if err == nil && cluster.cancelAfter == "prepare-reboot" { From ca1b0d3d328339326616ca98d734f4f2db05cdfe Mon Sep 17 00:00:00 2001 From: Bruno Andrade Date: Tue, 8 Sep 2026 16:30:42 -0300 Subject: [PATCH 6/6] ccoctl: Bound reboot MachineConfig create retries --- .../rotation/kubeadapter/reboot.go | 70 ++++++++++--------- .../kubeadapter/reboot_preflight_test.go | 41 +++++++++++ 2 files changed, 79 insertions(+), 32 deletions(-) diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/reboot.go b/pkg/cmd/provisioning/rotation/kubeadapter/reboot.go index 3c7f006616..2faaf89fb4 100644 --- a/pkg/cmd/provisioning/rotation/kubeadapter/reboot.go +++ b/pkg/cmd/provisioning/rotation/kubeadapter/reboot.go @@ -26,9 +26,10 @@ const ( rebootRecordNamePrefix = "cco-signer-key-rotation-reboot-" maxRebootRecordDataBytes = 900 * 1024 - rebootMachineConfigMaster = "95-cco-signer-key-rotation-reboot-master" - rebootMachineConfigWorker = "95-cco-signer-key-rotation-reboot-worker" - rebootMarkerPath = "/etc/kubernetes/cco-signer-key-rotation-reboot-id" + rebootMachineConfigMaster = "95-cco-signer-key-rotation-reboot-master" + rebootMachineConfigWorker = "95-cco-signer-key-rotation-reboot-worker" + rebootMarkerPath = "/etc/kubernetes/cco-signer-key-rotation-reboot-id" + maxRebootMachineConfigCreateAttempts = 3 managedAnnotation = "cloudcredential.openshift.io/signer-rotation-managed" rebootIDAnnotation = "cloudcredential.openshift.io/signer-rotation-reboot-id" @@ -509,37 +510,42 @@ func decodeRebootRecord(configMap *corev1.ConfigMap, guard rotation.RotationGuar } func (a *Adapter) ensureRebootMachineConfig(ctx context.Context, target, rebootID string) (bool, error) { - pool, err := a.resources.Get(ctx, machineConfigPoolGVR, target, metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("read MachineConfigPool %q: %w", target, err) - } - selectorLabels, err := machineConfigSelectorLabels(pool) - if err != nil { - return false, err - } - name := rebootMachineConfigName(target) - current, err := a.resources.Get(ctx, machineConfigGVR, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - desired := desiredRebootMachineConfig(nil, name, target, rebootID, selectorLabels) - _, createErr := a.resources.Create(ctx, machineConfigGVR, desired, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(createErr) { - return a.ensureRebootMachineConfig(ctx, target, rebootID) + for attempt := 1; ; attempt++ { + pool, err := a.resources.Get(ctx, machineConfigPoolGVR, target, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("read MachineConfigPool %q: %w", target, err) } - return createErr == nil, createErr - } - if err != nil { - return false, fmt.Errorf("read reboot MachineConfig %q: %w", name, err) - } - currentID, err := validateManagedRebootMachineConfig(current, target, selectorLabels) - if err != nil { - return false, err - } - if currentID == rebootID { - return false, nil + selectorLabels, err := machineConfigSelectorLabels(pool) + if err != nil { + return false, err + } + name := rebootMachineConfigName(target) + current, err := a.resources.Get(ctx, machineConfigGVR, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + desired := desiredRebootMachineConfig(nil, name, target, rebootID, selectorLabels) + _, createErr := a.resources.Create(ctx, machineConfigGVR, desired, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(createErr) { + if attempt >= maxRebootMachineConfigCreateAttempts { + return false, fmt.Errorf("create reboot MachineConfig %q still reported AlreadyExists after %d attempts: %w", name, attempt, createErr) + } + continue + } + return createErr == nil, createErr + } + if err != nil { + return false, fmt.Errorf("read reboot MachineConfig %q: %w", name, err) + } + currentID, err := validateManagedRebootMachineConfig(current, target, selectorLabels) + if err != nil { + return false, err + } + if currentID == rebootID { + return false, nil + } + desired := desiredRebootMachineConfig(current, name, target, rebootID, selectorLabels) + _, updateErr := a.resources.Update(ctx, machineConfigGVR, desired, metav1.UpdateOptions{}) + return updateErr == nil, updateErr } - desired := desiredRebootMachineConfig(current, name, target, rebootID, selectorLabels) - _, updateErr := a.resources.Update(ctx, machineConfigGVR, desired, metav1.UpdateOptions{}) - return updateErr == nil, updateErr } func (a *Adapter) validateExistingRebootMachineConfig(ctx context.Context, target string, selectorLabels map[string]string) error { diff --git a/pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go b/pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go index b2f155fc40..d4ee144994 100644 --- a/pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go +++ b/pkg/cmd/provisioning/rotation/kubeadapter/reboot_preflight_test.go @@ -327,6 +327,47 @@ func TestWaitUntilReturnsPromptlyWhenContextIsCancelled(t *testing.T) { } } +func TestEnsureRebootMachineConfigRecoversFromAlreadyExistsRace(t *testing.T) { + resources := newMemoryResourceClient(stableMachineConfigPool("master", 1)) + resources.beforeCreate = func(resource schema.GroupVersionResource, object *unstructured.Unstructured) error { + if resource != machineConfigGVR { + return nil + } + resources.putLocked(resource, object) + return apierrors.NewAlreadyExists(resource.GroupResource(), object.GetName()) + } + adapter := &Adapter{resources: resources} + + applied, err := adapter.ensureRebootMachineConfig(context.Background(), "master", testSingleTargetRebootIntent().ID) + if err != nil || applied { + t.Fatalf("ensureRebootMachineConfig() = %t, %v; want false, nil after observing the competing create", applied, err) + } + creates, updates := resources.actionCount(machineConfigGVR, rebootMachineConfigMaster) + if creates != 1 || updates != 0 { + t.Fatalf("MachineConfig mutations: create/update=%d/%d, want 1/0", creates, updates) + } +} + +func TestEnsureRebootMachineConfigBoundsAlreadyExistsRetries(t *testing.T) { + resources := newMemoryResourceClient(stableMachineConfigPool("master", 1)) + resources.beforeCreate = func(resource schema.GroupVersionResource, object *unstructured.Unstructured) error { + if resource != machineConfigGVR { + return nil + } + return apierrors.NewAlreadyExists(resource.GroupResource(), object.GetName()) + } + adapter := &Adapter{resources: resources} + + applied, err := adapter.ensureRebootMachineConfig(context.Background(), "master", testSingleTargetRebootIntent().ID) + if applied || !apierrors.IsAlreadyExists(err) || !strings.Contains(err.Error(), "after 3 attempts") { + t.Fatalf("ensureRebootMachineConfig() = %t, %v; want bounded AlreadyExists failure", applied, err) + } + creates, updates := resources.actionCount(machineConfigGVR, rebootMachineConfigMaster) + if creates != maxRebootMachineConfigCreateAttempts || updates != 0 { + t.Fatalf("MachineConfig mutations: create/update=%d/%d, want %d/0", creates, updates, maxRebootMachineConfigCreateAttempts) + } +} + func TestRequestRebootRecordsIntentBeforeMachineConfigsAndResumesMissingTarget(t *testing.T) { ctx := context.Background() guard := testGuardReference()