diff --git a/go.mod b/go.mod index c1d9f68fcf..6eea7b6765 100644 --- a/go.mod +++ b/go.mod @@ -136,3 +136,5 @@ require ( ) replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 + +replace github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260520063220-fea2dbe4012f diff --git a/go.sum b/go.sum index 04b9674b0b..5622434b71 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,6 @@ github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee h1:+S github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a h1:EKx2XhOKehd1C5ptY7IrLl4WV35E8kP0pRPnG5BUZXk= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a/go.mod h1:V933kvY/cb/Un7UCEOhXHUySNX327u7Epe8g9KNqg2Q= -github.com/openshift/library-go v0.0.0-20260513130945-942eeae4479f h1:fQ22ssXdHHmSj2nxz9ccJr7zx80oZa3HiL+Or7zS8VI= -github.com/openshift/library-go v0.0.0-20260513130945-942eeae4479f/go.mod h1:gKG9lctU0yEftSoT3DUyeIWz1oAgF0EHUpwI4pnCo4o= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 h1:PMTgifBcBRLJJiM+LgSzPDTk9/Rx4qS09OUrfpY6GBQ= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= @@ -215,6 +213,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tjungblu/library-go v0.0.0-20260520063220-fea2dbe4012f h1:Uj59W+q2NAKcjP3iv9K3jsHc9zMF5fMb3Pg0TRFte2c= +github.com/tjungblu/library-go v0.0.0-20260520063220-fea2dbe4012f/go.mod h1:gKG9lctU0yEftSoT3DUyeIWz1oAgF0EHUpwI4pnCo4o= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/test/e2e-encryption-rotation/encryption_rotation_test.go b/test/e2e-encryption-rotation/encryption_rotation_test.go index b2bb88795f..a0ecbf1312 100644 --- a/test/e2e-encryption-rotation/encryption_rotation_test.go +++ b/test/e2e-encryption-rotation/encryption_rotation_test.go @@ -1,7 +1,6 @@ package e2e_encryption_rotation import ( - "context" "flag" "fmt" "testing" @@ -10,7 +9,6 @@ import ( "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient" operatorencryption "github.com/openshift/cluster-kube-apiserver-operator/test/library/encryption" library "github.com/openshift/library-go/test/library/encryption" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var provider = flag.String("provider", "aescbc", "encryption provider used by the tests") @@ -32,15 +30,54 @@ func TestEncryptionRotation(t *testing.T) { CreateResourceFunc: operatorencryption.CreateAndStoreSecretOfLife, GetRawResourceFunc: operatorencryption.GetRawSecretOfLife, UnsupportedConfigFunc: func(raw []byte) error { - operatorClient := operatorencryption.GetOperator(t) - apiServerOperator, err := operatorClient.Get(context.TODO(), "cluster", metav1.GetOptions{}) - if err != nil { - return err - } - apiServerOperator.Spec.UnsupportedConfigOverrides.Raw = raw - _, err = operatorClient.Update(context.TODO(), apiServerOperator, metav1.UpdateOptions{}) - return err + return operatorencryption.UpdateUnsupportedConfig(t, raw) }, - EncryptionProvider: configv1.APIServerEncryption{Type: configv1.EncryptionType(*provider)}, + EncryptionProvider: library.EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionType(*provider)}}, + }) +} + +// TestEncryptionRotationDuringFirstMigration applies encryption (initial storage migration) and forces a key +// rotation while that first migration is still running. The cluster must converge to the last requested write key. +func TestEncryptionRotationDuringFirstMigration(t *testing.T) { + library.TestEncryptionRotationDuringFirstMigration(t, library.RotationScenario{ + BasicScenario: library.BasicScenario{ + Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, + EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace), + EncryptionConfigSecretNamespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + OperatorNamespace: operatorclient.OperatorNamespace, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertSecretsAndConfigMaps, + }, + CreateResourceFunc: operatorencryption.CreateAndStoreSecretOfLife, + GetRawResourceFunc: operatorencryption.GetRawSecretOfLife, + GetOperatorConditionsFunc: operatorencryption.GetClusterOperatorConditions, + UnsupportedConfigFunc: func(raw []byte) error { + return operatorencryption.UpdateUnsupportedConfig(t, raw) + }, + EncryptionProvider: library.EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionType(*provider)}}, + }) +} + +// TestEncryptionRotationDuringOngoingRotation forces a second key rotation while the migration triggered by the +// first forced rotation is still running. The cluster must converge to the last requested write key. +func TestEncryptionRotationDuringOngoingRotation(t *testing.T) { + library.TestEncryptionRotationDuringOngoingRotation(t, library.RotationScenario{ + BasicScenario: library.BasicScenario{ + Namespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + operatorclient.TargetNamespace, + EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", operatorclient.TargetNamespace), + EncryptionConfigSecretNamespace: operatorclient.GlobalMachineSpecifiedConfigNamespace, + OperatorNamespace: operatorclient.OperatorNamespace, + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertSecretsAndConfigMaps, + }, + CreateResourceFunc: operatorencryption.CreateAndStoreSecretOfLife, + GetRawResourceFunc: operatorencryption.GetRawSecretOfLife, + GetOperatorConditionsFunc: operatorencryption.GetClusterOperatorConditions, + UnsupportedConfigFunc: func(raw []byte) error { + return operatorencryption.UpdateUnsupportedConfig(t, raw) + }, + EncryptionProvider: library.EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionType(*provider)}}, }) } diff --git a/test/library/encryption/helpers.go b/test/library/encryption/helpers.go index e56a513707..821bf92860 100644 --- a/test/library/encryption/helpers.go +++ b/test/library/encryption/helpers.go @@ -6,6 +6,7 @@ import ( "testing" "time" + operatorv1 "github.com/openshift/api/operator/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -81,3 +82,23 @@ func SecretOfLife(t testing.TB, namespace string) runtime.Object { }, } } + +func UpdateUnsupportedConfig(t *testing.T, raw []byte) error { + operatorClient := GetOperator(t) + apiServerOperator, err := operatorClient.Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + apiServerOperator.Spec.UnsupportedConfigOverrides.Raw = raw + _, err = operatorClient.Update(context.TODO(), apiServerOperator, metav1.UpdateOptions{}) + return err +} + +func GetClusterOperatorConditions(t testing.TB) ([]operatorv1.OperatorCondition, error) { + operatorClient := GetOperator(t) + apiServerOperator, err := operatorClient.Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil { + return nil, err + } + return apiServerOperator.Status.Conditions, nil +} diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go b/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go index 326dccfd5e..3d367f271d 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/helpers.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/util/retry" configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" "github.com/openshift/library-go/test/library" @@ -36,9 +37,9 @@ var ( waitPollTimeout = 69*time.Minute + 10*time.Minute defaultEncryptionMode = string(configv1.EncryptionTypeIdentity) - SupportedStaticEncryptionProviders = []configv1.APIServerEncryption{ - {Type: configv1.EncryptionTypeAESGCM}, - {Type: configv1.EncryptionTypeAESCBC}, + SupportedStaticEncryptionProviders = []EncryptionProvider{ + {APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeAESGCM}}, + {APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeAESCBC}}, } ) @@ -56,28 +57,37 @@ type EncryptionKeyMeta struct { type UpdateUnsupportedConfigFunc func(raw []byte) error -func SetAndWaitForEncryptionType(t testing.TB, provider configv1.APIServerEncryption, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet { - t.Helper() - t.Logf("Starting encryption e2e test for %q mode", provider.Type) +// GetOperatorConditionsFuncType fetches operator conditions (e.g. for EncryptionMigrationControllerProgressing). +type GetOperatorConditionsFuncType func(t testing.TB) ([]operatorv1.OperatorCondition, error) +func SetAndWaitForEncryptionType(t testing.TB, provider EncryptionProvider, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet { clientSet := GetClients(t) lastMigratedKeyMeta, err := GetLastKeyMeta(t, clientSet.Kube, namespace, labelSelector) require.NoError(t, err) + ApplyAPIServerEncryptionType(t, clientSet, provider) + WaitForEncryptionKeyBasedOn(t, clientSet.Kube, lastMigratedKeyMeta, provider.Type, defaultTargetGRs, namespace, labelSelector) + return clientSet +} + +func ApplyAPIServerEncryptionType(t testing.TB, clientSet ClientSet, provider EncryptionProvider) { + t.Helper() + t.Logf("Starting encryption e2e test for %q mode", provider.Type) + apiServer, err := clientSet.ApiServerConfig.Get(context.TODO(), "cluster", metav1.GetOptions{}) require.NoError(t, err) - needsUpdate := !equality.Semantic.DeepEqual(apiServer.Spec.Encryption, provider) + needsUpdate := !equality.Semantic.DeepEqual(apiServer.Spec.Encryption, provider.APIServerEncryption) if needsUpdate { - t.Logf("Updating encryption configuration for APIServer from %#v to %#v", apiServer.Spec.Encryption, provider) - apiServer.Spec.Encryption = provider + if provider.Setup != nil { + provider.Setup(t) + } + t.Logf("Updating encryption configuration for APIServer from %#v to %#v", apiServer.Spec.Encryption, provider.APIServerEncryption) + apiServer.Spec.Encryption = provider.APIServerEncryption _, err = clientSet.ApiServerConfig.Update(context.TODO(), apiServer, metav1.UpdateOptions{}) require.NoError(t, err) } else { t.Logf("APIServer is already configured to use %q mode", provider.Type) } - - WaitForEncryptionKeyBasedOn(t, clientSet.Kube, lastMigratedKeyMeta, provider.Type, defaultTargetGRs, namespace, labelSelector) - return clientSet } func GetClients(t testing.TB) ClientSet { @@ -276,6 +286,27 @@ func ForceKeyRotation(t testing.TB, updateUnsupportedConfig UpdateUnsupportedCon }) } +// ClearForcedKeyRotationReason clears encryption.reason under UnsupportedConfigOverrides (same merge path as +// ForceKeyRotation). Call when a test finishes so the next test in sequence does not inherit a non-empty +// reason and the key controller does not keep seeing an external rotation request. +func ClearForcedKeyRotationReason(t testing.TB, updateUnsupportedConfig UpdateUnsupportedConfigFunc) error { + t.Helper() + t.Logf("Clearing forced encryption rotation reason (unsupported config overrides)") + data := map[string]map[string]string{ + "encryption": { + "reason": "", + }, + } + raw, err := json.Marshal(data) + if err != nil { + return err + } + + return onErrorWithTimeout(wait.ForeverTestTimeout, retry.DefaultBackoff, orError(errors.IsConflict, transientAPIError), func() error { + return updateUnsupportedConfig(raw) + }) +} + // hasResource returns whether the given group resource is contained in the migrated group resource list. func hasResource(expectedResource schema.GroupResource, actualResources []schema.GroupResource) bool { for _, gr := range actualResources { @@ -286,6 +317,117 @@ func hasResource(expectedResource schema.GroupResource, actualResources []schema return false } +const encryptionMigrationControllerProgressingType = "EncryptionMigrationControllerProgressing" + +// allTargetGRsMigrated reports whether every resource in targetGRs appears in meta's migrated list. +func allTargetGRsMigrated(meta EncryptionKeyMeta, targetGRs []schema.GroupResource) bool { + if len(targetGRs) == 0 { + return true + } + for _, gr := range targetGRs { + if !hasResource(gr, meta.Migrated) { + return false + } + } + return true +} + +// WaitUntilEncryptionStable waits until the latest write key secret reports the expected mode and all target resources are migrated. +func WaitUntilEncryptionStable(t testing.TB, kube kubernetes.Interface, expectedMode configv1.EncryptionType, targetGRs []schema.GroupResource, namespace, labelSelector string) { + t.Helper() + wantMode := string(expectedMode) + err := wait.Poll(waitPollInterval, waitPollTimeout, func() (bool, error) { + meta, err := GetLastKeyMeta(t, kube, namespace, labelSelector) + if err != nil { + return false, err + } + if meta.Mode != wantMode { + return false, nil + } + if !allTargetGRsMigrated(meta, targetGRs) { + return false, nil + } + return true, nil + }) + require.NoError(t, err) +} + +// WaitForNRotations waits until encryption is stable (expectedMode on the latest write key and every target +// group resource migrated), then asserts the latest write key secret's numeric suffix equals the baseline +// secret's suffix plus n. If baselineMeta.Name is empty (no prior write key), the baseline suffix is treated +// as 0. Use n to count new write-key revisions you expect after baselineMeta was captured (for example one +// per successful ForceKeyRotation, plus any additional revision from turning encryption on). +func WaitForNRotations(t testing.TB, kube kubernetes.Interface, expectedMode configv1.EncryptionType, targetGRs []schema.GroupResource, namespace, labelSelector string, baselineMeta EncryptionKeyMeta, n uint64) { + t.Helper() + WaitUntilEncryptionStable(t, kube, expectedMode, targetGRs, namespace, labelSelector) + + finalMeta, err := GetLastKeyMeta(t, kube, namespace, labelSelector) + require.NoError(t, err) + finalID, ok := EncryptionWriteKeySecretID(finalMeta.Name) + require.True(t, ok, "latest encryption key name must carry a numeric suffix: %q", finalMeta.Name) + + var baselineID uint64 + if len(baselineMeta.Name) == 0 { + baselineID = 0 + } else { + var baselineOK bool + baselineID, baselineOK = EncryptionWriteKeySecretID(baselineMeta.Name) + require.True(t, baselineOK, "baseline encryption key name must carry a numeric suffix: %q", baselineMeta.Name) + } + + expectedFinalID := baselineID + n + require.Equal(t, expectedFinalID, finalID, "expected write-key id %d (baseline id %d + %d), got final key %q id %d", expectedFinalID, baselineID, n, finalMeta.Name, finalID) +} + +// EncryptionWriteKeySecretID returns the numeric suffix of an encryption write-key secret name (the value +// used when sorting keys by revision, e.g. encryption-key-openshift-apiserver-4 yields 4). +func EncryptionWriteKeySecretID(secretName string) (uint64, bool) { + return encryptionKeyNameToKeyID(secretName) +} + +// WaitForEncryptionMigrationInProgressWindow waits until storage migration is actively running so another +// encryption change can be stacked. It returns false when the migration for expectedWriteKey completed +// before an in-progress snapshot could be observed (caller may t.Skip). +func WaitForEncryptionMigrationInProgressWindow(t testing.TB, kube kubernetes.Interface, getOp GetOperatorConditionsFuncType, expectedWriteKey string, targetGRs []schema.GroupResource, namespace, labelSelector string) bool { + t.Helper() + const kubePoll = 1 * time.Second + const windowWait = 25 * time.Minute + + if getOp != nil { + err := wait.Poll(2*time.Second, windowWait, func() (bool, error) { + conds, err := getOp(t) + if err != nil { + return false, err + } + for _, c := range conds { + if c.Type == encryptionMigrationControllerProgressingType && c.Status == operatorv1.ConditionTrue && c.Reason == "Migrating" { + return true, nil + } + } + return false, nil + }) + if err == nil { + return true + } + t.Logf("encryption migration progressing condition not observed within %v, falling back to secret metadata polling: %v", windowWait, err) + } + + deadline := time.Now().Add(windowWait) + for time.Now().Before(deadline) { + meta, err := GetLastKeyMeta(t, kube, namespace, labelSelector) + require.NoError(t, err) + if meta.Name == expectedWriteKey { + if allTargetGRsMigrated(meta, targetGRs) { + return false + } + return true + } + time.Sleep(kubePoll) + } + require.FailNow(t, fmt.Sprintf("timed out after %v waiting for migration in progress on key %q", windowWait, expectedWriteKey)) + return false +} + func encryptionKeyNameToKeyID(name string) (uint64, bool) { lastIdx := strings.LastIndex(name, "-") idString := name diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/k8s_mock_kms_plugin_deployer.go b/vendor/github.com/openshift/library-go/test/library/encryption/kms/k8s_mock_kms_plugin_deployer.go index 60aebe22dd..27729b5b39 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/kms/k8s_mock_kms_plugin_deployer.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/k8s_mock_kms_plugin_deployer.go @@ -29,7 +29,7 @@ const ( WellKnownUpstreamMockKMSPluginNamespace = "k8s-mock-plugin" // WellKnownUpstreamMockKMSPluginImage is the pre-built mock KMS plugin image. - WellKnownUpstreamMockKMSPluginImage = "quay.io/openshifttest/mock-kms-plugin@sha256:76444d7e37d0d2d0f4dfae31893e937f766439ce07036bf19325050b594d2e2c" + WellKnownUpstreamMockKMSPluginImage = "quay.io/openshifttest/mock-kms-plugin@sha256:03bb07a2c08b509653c4c70217a06a4b389c10b4d87922f50ee5eac82db5e140" // DefaultKMSPluginCount is the default number of KMS plugin instances to deploy. DefaultKMSPluginCount = 10 diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go b/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go index ff09697334..987293a0a6 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/kms/vault.go @@ -1,11 +1,70 @@ package kms import ( + "context" + "fmt" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/clock" + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" + library "github.com/openshift/library-go/test/library/encryption" +) + +const ( + defaultVaultNamespace = "vault-kms" + defaultVaultPodName = "vault-0" + defaultVaultCredentialsSecret = "vault-credentials" + defaultVaultAppRoleSecretName = "vault-approle-secret" + defaultVaultKMSPluginImage = "quay.io/openshifttest/mock-kms-plugin@sha256:03bb07a2c08b509653c4c70217a06a4b389c10b4d87922f50ee5eac82db5e140" + defaultVaultAddress = "https://vault.vault-kms.svc:8200" + defaultVaultEnterpriseNS = "admin" + defaultVaultTransitMount = "transit" + defaultVaultTransitKey = "kms-key" + defaultAppRoleTargetNamespace = "openshift-config" + vaultCommandTimeout = 30 * time.Second ) -// DefaultFakeKMSPluginConfig is a fake Vault KMS configuration used by tests. -// The values are not real and are likely to change as the KMS integration evolves. +// DefaultVaultEncryptionProvider is a ready-to-use Vault KMS EncryptionProvider for e2e tests. +// It bundles the default config with the AppRole secret setup. +var DefaultVaultEncryptionProvider = library.EncryptionProvider{ + APIServerEncryption: DefaultVaultKMSPluginConfig, + Setup: ensureDefaultVaultAppRoleSecret, +} + +// DefaultVaultKMSPluginConfig is the standard Vault KMS encryption config +// used by CI e2e tests. +var DefaultVaultKMSPluginConfig = configv1.APIServerEncryption{ + Type: configv1.EncryptionTypeKMS, + KMS: configv1.KMSPluginConfig{ + Type: configv1.VaultKMSProvider, + Vault: configv1.VaultKMSPluginConfig{ + KMSPluginImage: defaultVaultKMSPluginImage, + VaultAddress: defaultVaultAddress, + VaultNamespace: defaultVaultEnterpriseNS, + TransitMount: defaultVaultTransitMount, + TransitKey: defaultVaultTransitKey, + Authentication: configv1.VaultAuthentication{ + Type: configv1.VaultAuthenticationTypeAppRole, + AppRole: configv1.VaultAppRoleAuthentication{ + Secret: configv1.VaultSecretReference{Name: defaultVaultAppRoleSecretName}, + }, + }, + }, + }, +} + +// DefaultFakeKMSPluginConfig is a fake Vault KMS configuration used by unit tests. var DefaultFakeKMSPluginConfig = configv1.KMSPluginConfig{ Type: configv1.VaultKMSProvider, Vault: configv1.VaultKMSPluginConfig{ @@ -20,3 +79,91 @@ var DefaultFakeKMSPluginConfig = configv1.KMSPluginConfig{ TransitKey: "test-transit-key", }, } + +// ensureDefaultVaultAppRoleSecret reads credentials from the vault-credentials secret +// (created by a CI step) and applies the AppRole secret in openshift-config +// using the default configuration constants. +func ensureDefaultVaultAppRoleSecret(t testing.TB) { + t.Helper() + ctx := t.Context() + cs := library.GetClients(t) + + creds, err := cs.Kube.CoreV1().Secrets(defaultVaultNamespace).Get(ctx, defaultVaultCredentialsSecret, metav1.GetOptions{}) + require.NoError(t, err, "failed to read %s/%s secret (was the vault-install CI step run?)", defaultVaultNamespace, defaultVaultCredentialsSecret) + + required := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultVaultAppRoleSecretName, + Namespace: defaultAppRoleTargetNamespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "roleID": creds.Data["role-id"], + "secretID": creds.Data["secret-id"], + }, + } + recorder := events.NewInMemoryRecorder("vault-approle-secret-setup", clock.RealClock{}) + _, changed, err := resourceapply.ApplySecret(ctx, cs.Kube.CoreV1(), recorder, required) + require.NoError(t, err, "failed to apply AppRole secret") + t.Logf("Applied AppRole secret %s in %s (changed=%v)", defaultVaultAppRoleSecretName, defaultAppRoleTargetNamespace, changed) +} + +// RotateVaultTransitKey rotates the Vault transit encryption key. All old key versions are retained. +// Reference: https://developer.hashicorp.com/vault/api-docs/secret/transit#rotate-key +// Steps: +// 1. Get initial key version +// 2. Execute 'vault write -f transit/keys//rotate' via oc exec +// 3. Get new key version and validate it increased +func RotateVaultTransitKey(t testing.TB) { + t.Helper() + ctx := t.Context() + + initialVersion := getCurrentKeyVersion(ctx, t) + rotateKey(ctx, t) + newVersion := getCurrentKeyVersion(ctx, t) + + require.Greater(t, newVersion, initialVersion, "rotation failed: version did not increase (before=%d, after=%d)", initialVersion, newVersion) +} + +// rotateKey executes the vault key rotation command +func rotateKey(ctx context.Context, t testing.TB) { + t.Helper() + commandCtx, cancel := context.WithTimeout(ctx, vaultCommandTimeout) + defer cancel() + + // Command: vault write -f transit/keys//rotate + // Reference: https://developer.hashicorp.com/vault/api-docs/secret/transit#rotate-key + cmd := exec.CommandContext(commandCtx, "oc", "exec", defaultVaultPodName, "-n", defaultVaultNamespace, "--", + "vault", "write", "-f", fmt.Sprintf("transit/keys/%s/rotate", defaultVaultTransitKey)) + + t.Logf("Executing: %s", cmd.String()) + output, err := cmd.Output() + if ee, ok := err.(*exec.ExitError); ok { + require.NoError(t, err, "vault key rotation failed, stderr: %s", string(ee.Stderr)) + } + require.NoError(t, err, "vault key rotation failed") + t.Logf("Command output: %s", string(output)) +} + +// getCurrentKeyVersion retrieves the current (latest) key version +func getCurrentKeyVersion(ctx context.Context, t testing.TB) int { + t.Helper() + commandCtx, cancel := context.WithTimeout(ctx, vaultCommandTimeout) + defer cancel() + + cmd := exec.CommandContext(commandCtx, "oc", "exec", defaultVaultPodName, "-n", defaultVaultNamespace, "--", + "vault", "read", "-field=latest_version", fmt.Sprintf("transit/keys/%s", defaultVaultTransitKey)) + + t.Logf("Executing: %s", cmd.String()) + output, err := cmd.Output() + if ee, ok := err.(*exec.ExitError); ok { + require.NoError(t, err, "failed to read key version, stderr: %s", string(ee.Stderr)) + } + require.NoError(t, err, "failed to read key version") + t.Logf("Command output: %s", string(output)) + + version, err := strconv.Atoi(strings.TrimSpace(string(output))) + require.NoError(t, err, "failed to parse key version from output: %q", string(output)) + + return version +} diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go b/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go index 0095f20704..71b268a1e4 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/perf_scenarios.go @@ -5,11 +5,8 @@ import ( "time" configv1 "github.com/openshift/api/config/v1" - v1 "github.com/openshift/api/operator/v1" ) -type GetOperatorConditionsFuncType func(t testing.TB) ([]v1.OperatorCondition, error) - type PerfScenario struct { BasicScenario GetOperatorConditionsFunc GetOperatorConditionsFuncType @@ -19,7 +16,7 @@ type PerfScenario struct { AssertMigrationTime func(t testing.TB, migrationTime time.Duration) // DBLoaderWorker is the number of workers that will execute DBLoaderFunc DBLoaderWorkers int - EncryptionProvider configv1.APIServerEncryption + EncryptionProvider EncryptionProvider } func TestPerfEncryption(t *testing.T, scenario PerfScenario) { diff --git a/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go b/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go index 7b2a154e62..e0f29c5f85 100644 --- a/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go +++ b/vendor/github.com/openshift/library-go/test/library/encryption/scenarios.go @@ -25,19 +25,27 @@ type BasicScenario struct { AssertFunc func(t testing.TB, clientSet ClientSet, expectedMode configv1.EncryptionType, namespace, labelSelector string) } +// EncryptionProvider pairs an encryption config with an optional setup function +// that ensures prerequisites (secrets, credentials, infrastructure) are in place. +type EncryptionProvider struct { + configv1.APIServerEncryption + // Setup is called once before the provider is first used. May be nil. + Setup func(t testing.TB) +} + func TestEncryptionTypeIdentity(t testing.TB, scenario BasicScenario) { e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) - clientSet := SetAndWaitForEncryptionType(e, configv1.APIServerEncryption{Type: configv1.EncryptionTypeIdentity}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + clientSet := SetAndWaitForEncryptionType(e, EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeIdentity}}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, configv1.EncryptionTypeIdentity, scenario.Namespace, scenario.LabelSelector) } func TestEncryptionTypeUnset(t testing.TB, scenario BasicScenario) { e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) - clientSet := SetAndWaitForEncryptionType(e, configv1.APIServerEncryption{}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + clientSet := SetAndWaitForEncryptionType(e, EncryptionProvider{}, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) scenario.AssertFunc(e, clientSet, configv1.EncryptionTypeIdentity, scenario.Namespace, scenario.LabelSelector) } -func resolveProvider(t testing.TB, defaultType configv1.EncryptionType, providers []configv1.APIServerEncryption) configv1.APIServerEncryption { +func resolveProvider(t testing.TB, defaultType configv1.EncryptionType, providers []EncryptionProvider) EncryptionProvider { t.Helper() if len(providers) > 1 { t.Fatalf("expected at most one provider, got %d", len(providers)) @@ -45,10 +53,10 @@ func resolveProvider(t testing.TB, defaultType configv1.EncryptionType, provider if len(providers) == 1 { return providers[0] } - return configv1.APIServerEncryption{Type: defaultType} + return EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: defaultType}} } -func TestEncryptionTypeAESCBC(t testing.TB, scenario BasicScenario, providers ...configv1.APIServerEncryption) { +func TestEncryptionTypeAESCBC(t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { provider := resolveProvider(t, configv1.EncryptionTypeAESCBC, providers) e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) clientSet := SetAndWaitForEncryptionType(e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) @@ -56,7 +64,7 @@ func TestEncryptionTypeAESCBC(t testing.TB, scenario BasicScenario, providers .. AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) } -func TestEncryptionTypeAESGCM(t testing.TB, scenario BasicScenario, providers ...configv1.APIServerEncryption) { +func TestEncryptionTypeAESGCM(t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { provider := resolveProvider(t, configv1.EncryptionTypeAESGCM, providers) e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) clientSet := SetAndWaitForEncryptionType(e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) @@ -64,7 +72,7 @@ func TestEncryptionTypeAESGCM(t testing.TB, scenario BasicScenario, providers .. AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) } -func TestEncryptionTypeKMS(t testing.TB, scenario BasicScenario, providers ...configv1.APIServerEncryption) { +func TestEncryptionTypeKMS(t testing.TB, scenario BasicScenario, providers ...EncryptionProvider) { provider := resolveProvider(t, configv1.EncryptionTypeKMS, providers) e := NewE(t, PrintEventsOnFailure(scenario.OperatorNamespace)) clientSet := SetAndWaitForEncryptionType(e, provider, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) @@ -72,7 +80,7 @@ func TestEncryptionTypeKMS(t testing.TB, scenario BasicScenario, providers ...co AssertEncryptionConfig(e, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) } -func TestEncryptionType(t testing.TB, scenario BasicScenario, provider configv1.APIServerEncryption) { +func TestEncryptionType(t testing.TB, scenario BasicScenario, provider EncryptionProvider) { switch provider.Type { case configv1.EncryptionTypeAESCBC: TestEncryptionTypeAESCBC(t, scenario, provider) @@ -94,7 +102,7 @@ type OnOffScenario struct { AssertResourceNotEncryptedFunc func(t testing.TB, clientSet ClientSet, resource runtime.Object) ResourceFunc func(t testing.TB, namespace string) runtime.Object ResourceName string - EncryptionProvider configv1.APIServerEncryption + EncryptionProvider EncryptionProvider } type testStep struct { @@ -155,13 +163,13 @@ type ProvidersMigrationScenario struct { // EncryptionProviders is the list of encryption providers to migrate through. // The test will migrate through each provider in order, then always end by // switching to identity (off) to verify the resource is re-written unencrypted. - EncryptionProviders []configv1.APIServerEncryption + EncryptionProviders []EncryptionProvider } // ShuffleEncryptionProviders returns a new slice with the providers in random order, // leaving the original slice unchanged. Use this to test different migration orderings. -func ShuffleEncryptionProviders(providers []configv1.APIServerEncryption) []configv1.APIServerEncryption { - shuffled := make([]configv1.APIServerEncryption, len(providers)) +func ShuffleEncryptionProviders(providers []EncryptionProvider) []EncryptionProvider { + shuffled := make([]EncryptionProvider, len(providers)) copy(shuffled, providers) mathrand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] @@ -232,7 +240,10 @@ type RotationScenario struct { CreateResourceFunc func(t testing.TB, clientSet ClientSet, namespace string) runtime.Object GetRawResourceFunc func(t testing.TB, clientSet ClientSet, namespace string) string UnsupportedConfigFunc UpdateUnsupportedConfigFunc - EncryptionProvider configv1.APIServerEncryption + EncryptionProvider EncryptionProvider + // GetOperatorConditionsFunc is optional. Overlap tests use it to detect an active migration via + // EncryptionMigrationControllerProgressing before falling back to polling encryption key secrets. + GetOperatorConditionsFunc GetOperatorConditionsFuncType } // TestEncryptionRotation first encrypts data with aescbc key @@ -244,6 +255,14 @@ func TestEncryptionRotation(t testing.TB, scenario RotationScenario) { // step 1: create the desired resource e := NewE(t) + defer func() { + if err := ClearForcedKeyRotationReason(e, scenario.UnsupportedConfigFunc); err != nil { + e.Logf("cleanup: clear encryption rotation reason: %v", err) + if !t.Failed() { + require.NoError(e, err, "test cleanup: clear encryption rotation reason") + } + } + }() clientSet := GetClients(e) scenario.CreateResourceFunc(e, GetClients(e), ns) @@ -268,3 +287,84 @@ func TestEncryptionRotation(t testing.TB, scenario RotationScenario) { // TODO: assert conditions - operator and encryption migration controller must report status as active not progressing, and not failing for all scenarios } + +// TestEncryptionRotationDuringFirstMigration ensures storage starts from identity, turns encryption on +// (initial migration), forces a key rotation while that first migration is still running, then asserts +// convergence. Use this to exercise overlap between the first encrypt migration and an external rotation +// reason—not stacked rotations on an already-encrypted cluster. +func TestEncryptionRotationDuringFirstMigration(t testing.TB, scenario RotationScenario) { + ns := scenario.Namespace + labelSelector := scenario.LabelSelector + + e := NewE(t) + defer func() { + if err := ClearForcedKeyRotationReason(e, scenario.UnsupportedConfigFunc); err != nil { + e.Logf("cleanup: clear encryption rotation reason: %v", err) + if !t.Failed() { + require.NoError(e, err, "test cleanup: clear encryption rotation reason") + } + } + }() + clientSet := GetClients(e) + scenario.CreateResourceFunc(e, clientSet, ns) + + // ApplyAPIServerEncryptionType is a no-op when the APIServer is already on the target type; start from + // identity so the first storage migration always runs. + TestEncryptionTypeIdentity(t, scenario.BasicScenario) + + prevMeta, err := GetLastKeyMeta(e, clientSet.Kube, ns, labelSelector) + require.NoError(e, err) + expectedFirstWriteKey, err := determineNextEncryptionKeyName(prevMeta.Name, labelSelector) + require.NoError(e, err) + + ApplyAPIServerEncryptionType(e, clientSet, scenario.EncryptionProvider) + + if !WaitForEncryptionMigrationInProgressWindow(e, clientSet.Kube, scenario.GetOperatorConditionsFunc, expectedFirstWriteKey, scenario.TargetGRs, ns, labelSelector) { + t.Skipf("initial migration finished before an in-progress window was observed; set GetOperatorConditionsFunc or use a cluster where migration stays visible longer") + } + + require.NoError(e, ForceKeyRotation(e, scenario.UnsupportedConfigFunc, fmt.Sprintf("test-rotation-during-first-migration-%s", rand.String(4)))) + // n=2: one write-key revision from turning encryption on, one from ForceKeyRotation. + WaitForNRotations(e, clientSet.Kube, scenario.EncryptionProvider.Type, scenario.TargetGRs, ns, labelSelector, prevMeta, 2) + + scenario.AssertFunc(e, clientSet, scenario.EncryptionProvider.Type, ns, labelSelector) +} + +// TestEncryptionRotationDuringOngoingRotation runs with encryption already enabled and stable, then forces +// two key rotations in quick succession so the second happens while migration from the first is still +// in progress. This targets stacked external rotation reasons—not the first encrypt-from-identity path. +func TestEncryptionRotationDuringOngoingRotation(t testing.TB, scenario RotationScenario) { + ns := scenario.Namespace + labelSelector := scenario.LabelSelector + + e := NewE(t) + defer func() { + if err := ClearForcedKeyRotationReason(e, scenario.UnsupportedConfigFunc); err != nil { + e.Logf("cleanup: clear encryption rotation reason: %v", err) + if !t.Failed() { + require.NoError(e, err, "test cleanup: clear encryption rotation reason") + } + } + }() + clientSet := GetClients(e) + scenario.CreateResourceFunc(e, clientSet, ns) + + TestEncryptionType(t, scenario.BasicScenario, scenario.EncryptionProvider) + + metaAfterEncrypt, err := GetLastKeyMeta(e, clientSet.Kube, ns, labelSelector) + require.NoError(e, err) + expectedNextWriteKey, err := determineNextEncryptionKeyName(metaAfterEncrypt.Name, labelSelector) + require.NoError(e, err) + + require.NoError(e, ForceKeyRotation(e, scenario.UnsupportedConfigFunc, fmt.Sprintf("test-rotation-overlap-first-%s", rand.String(4)))) + + if !WaitForEncryptionMigrationInProgressWindow(e, clientSet.Kube, scenario.GetOperatorConditionsFunc, expectedNextWriteKey, scenario.TargetGRs, ns, labelSelector) { + t.Skipf("migration after first forced rotation finished before an in-progress window was observed; set GetOperatorConditionsFunc or use a slower cluster") + } + + require.NoError(e, ForceKeyRotation(e, scenario.UnsupportedConfigFunc, fmt.Sprintf("test-rotation-overlap-second-%s", rand.String(4)))) + // n=2: two ForceKeyRotation steps after metaAfterEncrypt. + WaitForNRotations(e, clientSet.Kube, scenario.EncryptionProvider.Type, scenario.TargetGRs, ns, labelSelector, metaAfterEncrypt, 2) + + scenario.AssertFunc(e, clientSet, scenario.EncryptionProvider.Type, ns, labelSelector) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 5cf515f50c..dfbc9b1954 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -407,7 +407,7 @@ github.com/openshift/client-go/security/informers/externalversions/internalinter github.com/openshift/client-go/security/informers/externalversions/security github.com/openshift/client-go/security/informers/externalversions/security/v1 github.com/openshift/client-go/security/listers/security/v1 -# github.com/openshift/library-go v0.0.0-20260513130945-942eeae4479f +# github.com/openshift/library-go v0.0.0-20260513130945-942eeae4479f => github.com/tjungblu/library-go v0.0.0-20260520063220-fea2dbe4012f ## explicit; go 1.25.0 github.com/openshift/library-go/pkg/apiserver/jsonpatch github.com/openshift/library-go/pkg/assets @@ -1699,3 +1699,4 @@ sigs.k8s.io/structured-merge-diff/v6/value ## explicit; go 1.22 sigs.k8s.io/yaml # github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 +# github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260520063220-fea2dbe4012f