From 15007c4742c31184fcc83809cedce124a2b629b7 Mon Sep 17 00:00:00 2001 From: OAharoni-RedHat Date: Tue, 1 Sep 2026 15:29:42 -0400 Subject: [PATCH 1/2] CI test infrastructure; fix namespaces, subscriptions, and gitignore --- .gitignore | 3 + Makefile | 1 + overrides/values-cluster-names.yaml | 2 +- overrides/values-odf-chart.yaml | 21 ++ pattern-metadata.yaml | 4 +- tests/ci/README.md | 32 +++ tests/ci/__init__.py | 1 + tests/ci/conftest.py | 1 + tests/ci/requirements.txt | 15 ++ tests/ci/test_odf.py | 315 ++++++++++++++++++++++++++++ variants/odf/values-odf.yaml | 13 ++ 11 files changed, 405 insertions(+), 3 deletions(-) create mode 100644 tests/ci/README.md create mode 100644 tests/ci/__init__.py create mode 100644 tests/ci/conftest.py create mode 100644 tests/ci/requirements.txt create mode 100644 tests/ci/test_odf.py diff --git a/.gitignore b/.gitignore index 2a7ce4a0..d86a8d1a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ super-linter.log .ansible /ocp-primary.yaml /ocp-secondary.yaml +tests/ci/.results/ +tests/ci/.pytest_cache/ +tests/ci/__pycache__/ \ No newline at end of file diff --git a/Makefile b/Makefile index bb24d056..f7ec6d31 100644 --- a/Makefile +++ b/Makefile @@ -10,3 +10,4 @@ install-byoc: ramen-prereq pattern-install ## Installs the pattern onto a cluste ramen-prereq: ## Check if values.byoc false do nothing, else run the precheck against clusters accessed from values-secrets echo "Running precheck for ramendr" cd ansible && ansible-playbook -i hosts $(EXTRA_ARGS) $(EXTRA_VARS) playbooks/validate_byoc.yml + diff --git a/overrides/values-cluster-names.yaml b/overrides/values-cluster-names.yaml index e8f22ade..239d6424 100644 --- a/overrides/values-cluster-names.yaml +++ b/overrides/values-cluster-names.yaml @@ -34,4 +34,4 @@ drpc: # ./pattern.sh make install-byoc. # That target will run a validation play to ensure the cluster # setup will work with ramen patterns -byoc: false +byoc: true \ No newline at end of file diff --git a/overrides/values-odf-chart.yaml b/overrides/values-odf-chart.yaml index 72b80155..de35d040 100644 --- a/overrides/values-odf-chart.yaml +++ b/overrides/values-odf-chart.yaml @@ -7,3 +7,24 @@ storageSystem: objectStorage: enable: true + +odf: + # Workaround for a Rook/ODF bug: the default zone topology spread constraint on OSD + # prepare pods requires device-class=ssd, a label only added after OSD initialization, + # not during prepare. The constraint matches zero pods and is a no-op, allowing all + # 3 prepare pods to land on the same node. This sets a hard per-hostname constraint + # that actually matches rook-ceph-osd-prepare pods, ensuring one prepare pod per node. + # Requires openshift-data-foundations chart >= 0.3.2 (exposes odf.preparePlacement). + # When using OPP chart <= 0.3.1, the odf-prepare-patch ArgoCD app applies this patch + # as a PostSync hook instead. + preparePlacement: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-osd-prepare diff --git a/pattern-metadata.yaml b/pattern-metadata.yaml index a19c6b52..bb1e98eb 100644 --- a/pattern-metadata.yaml +++ b/pattern-metadata.yaml @@ -20,12 +20,12 @@ requirements: platform: aws: replicas: 3 - type: m5.2xlarge + type: m5.4xlarge controlPlane: platform: aws: replicas: 3 - type: m5.xlarge + type: m5.4xlarge spoke: # Managed DR clusters (primary/secondary) compute: platform: diff --git a/tests/ci/README.md b/tests/ci/README.md new file mode 100644 index 00000000..d39ca7cf --- /dev/null +++ b/tests/ci/README.md @@ -0,0 +1,32 @@ +# CI Tests for ramendr-starter-kit + +Pytest is invoked per-clustergroup from the repository root via the `interop-test` Tekton task: + +```console +pytest -lv test_.py --junit-xml .results/test_.xml +``` + +## Required environment variables + +| Variable | Description | +|---|---| +| `VP_HUBCONFIG` | Path to kubeconfig for the hub cluster | +| `VP_SPOKECONFIG` | Path to kubeconfig for `ocp-primary` (DR spoke 1) | +| `VP_SPOKECONFIG_SECONDARY` | Path to kubeconfig for `ocp-secondary` (DR spoke 2) | +| `TARGET_CLUSTERGROUP` | Clustergroup to test (defaults to `hub`) | + +## Running tests locally + +From the repository root: + +```bash +VP_HUBCONFIG=~/.kube/hub-config \ +VP_SPOKECONFIG=~/.kube/spoke-primary-config \ +VP_SPOKECONFIG_SECONDARY=~/.kube/spoke-secondary-config \ +TARGET_CLUSTERGROUP=odf \ +./pattern.sh make run-ci-tests +``` + +## Test coverage + +- `test_odf.py` — hub-side operator subscriptions, pod health, ACM ManagedCluster status, ArgoCD reachability and application health for all three clusters diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 00000000..3dc1f76b --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/tests/ci/conftest.py b/tests/ci/conftest.py new file mode 100644 index 00000000..a531ae25 --- /dev/null +++ b/tests/ci/conftest.py @@ -0,0 +1 @@ +from validatedpatterns_tests.interop.conftest_openshift import * # noqa diff --git a/tests/ci/requirements.txt b/tests/ci/requirements.txt new file mode 100644 index 00000000..0a68e1e2 --- /dev/null +++ b/tests/ci/requirements.txt @@ -0,0 +1,15 @@ +ansible-core==2.18.* +ansible-runner +awxkit +kubernetes +openshift +boto3>=1.21 +botocore>=1.24 +awscli>=1.22 +azure-cli>=2.34 +gcloud +humanize +pytz +pytest +junitparser +vp-qe-test-common @ git+https://github.com/validatedpatterns/vp-qe-test-common.git@v1 \ No newline at end of file diff --git a/tests/ci/test_odf.py b/tests/ci/test_odf.py new file mode 100644 index 00000000..4f40434a --- /dev/null +++ b/tests/ci/test_odf.py @@ -0,0 +1,315 @@ +import pytest +import validatedpatterns_tests.interop.application as application +import validatedpatterns_tests.interop.components as components +import validatedpatterns_tests.interop.subscription as subscription + +# ── DR helpers ─────────────────────────────────────────────────────────────── + + +def _get_condition(conditions, cond_type): + for c in conditions or []: + if c.get("type") == cond_type: + return c + return None + + +def _condition_status(conditions, cond_type): + c = _get_condition(conditions, cond_type) + if c is None: + return None + return c.get("status") == "True" + + +def _list_resource(client, api_version, kind): + resource = client.resources.get(api_version=api_version, kind=kind) + return resource.get().items + + +# ── Hub subscriptions ──────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_subscription_status_hub(openshift_dyn_client): + expected_subs = { + "openshift-gitops-operator": ["openshift-gitops-operator"], + "advanced-cluster-management": ["open-cluster-management"], + "multicluster-engine": ["multicluster-engine"], + # odf-multicluster-orchestrator installs into + # openshift-operators (cluster-wide) + "odf-multicluster-orchestrator": ["openshift-operators"], + # cert-manager is managed by the RH operator + # (openshift-cert-manager-operator) + "openshift-cert-manager-operator": ["cert-manager-operator"], + } + subscription.assert_subscription_status( + openshift_dyn_client, + expected_subs, + ) + + +# ── Spoke subscriptions ────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_SPOKECONFIG", "VP_SPOKECONFIG_SECONDARY"], + indirect=True, +) +def test_subscription_status_spoke(openshift_dyn_client): + expected_subs = { + "openshift-gitops-operator": ["openshift-gitops-operator"], + "odf-operator": ["openshift-storage"], + "kubevirt-hyperconverged": ["openshift-cnv"], + # OADP subscription name is redhat-oadp-operator on spoke + "redhat-oadp-operator": ["openshift-adp"], + # Ramen DR cluster operator on spoke + "ramen-dr-cluster-subscription": ["openshift-dr-system"], + } + subscription.assert_subscription_status( + openshift_dyn_client, + expected_subs, + ) + + +# ── Hub pod health ─────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_pod_status_hub(openshift_dyn_client): + # odf-multicluster-orchestrator and ramen DR hub pods + # live in openshift-operators (cluster-wide); + # we check that namespace for the whole operator ecosystem + projects = [ + "patterns-operator", + "open-cluster-management", + "open-cluster-management-hub", + "cert-manager", + "cert-manager-operator", + "cluster-ca-mgt", + "vault", + "vp-gitops", + "external-secrets", + ] + components.assert_pod_status(openshift_dyn_client, projects, skip_check=[]) + + +# ── Spoke pod health ───────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_SPOKECONFIG", "VP_SPOKECONFIG_SECONDARY"], + indirect=True, +) +def test_pod_status_spoke(openshift_dyn_client): + # openshift-storage is intentionally excluded: ODF spawns + # short-lived CronJob pods (storageclient-*-status-reporter-*) + # that get garbage-collected between list and per-pod GET, + # causing a spurious 404. ODF health on spokes is covered by + # test_subscription_status_spoke and + # test_drpc_available_and_peer_ready. + projects = [ + "open-cluster-management-agent", + "open-cluster-management-agent-addon", + "openshift-cnv", + "openshift-adp", + "openshift-dr-system", + "vp-gitops", + "external-secrets", + ] + components.assert_pod_status(openshift_dyn_client, projects, skip_check=[]) + + +# ── ACM ManagedCluster status ──────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_managed_clusters(openshift_dyn_client): + # Both DR clusters must be imported and available in ACM + components.assert_managed_clusters( + openshift_dyn_client, + ["ocp-primary", "ocp-secondary"], + ) + + +# ── ArgoCD reachability ────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG", "VP_SPOKECONFIG", "VP_SPOKECONFIG_SECONDARY"], + indirect=True, +) +def test_argocd_reachable(openshift_dyn_client): + components.assert_argocd_reachable(openshift_dyn_client) + + +# ── ArgoCD application health ──────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_argocd_applications_health_hub(openshift_dyn_client): + # vp-gitops: top-level hub app. + # ramendr-starter-kit-odf: component apps. + # vp-manage-proxy-cluster-ca OutOfSync: ESO schema defaults + # (nullBytePolicy, deletionPolicy, engineVersion, mergePolicy) + # not emitted by the chart; fixed via + # ignoreDifferences in values-odf.yaml. + projects = ["vp-gitops", "ramendr-starter-kit-odf"] + application.assert_argocd_applications(openshift_dyn_client, projects) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_SPOKECONFIG", "VP_SPOKECONFIG_SECONDARY"], + indirect=True, +) +def test_argocd_applications_health_spoke(openshift_dyn_client): + projects = ["vp-gitops"] + application.assert_argocd_applications(openshift_dyn_client, projects) + + +# ── DR control-plane health ────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_drpolicy_validated(openshift_dyn_client): + """All DRPolicy objects must have Validated=True.""" + policies = _list_resource( + openshift_dyn_client, + api_version="ramendr.openshift.io/v1alpha1", + kind="DRPolicy", + ) + assert policies, "No DRPolicy objects found on hub" + + failures = [] + for p in policies: + name = p.metadata.name + conditions = (p.status or {}).get("conditions", []) + if not _condition_status(conditions, "Validated"): + c = _get_condition(conditions, "Validated") + reason = c.get("reason", "unknown") if c else "condition missing" + message = c.get("message", "") if c else "" + failures.append(f"{name}: Validated != True ({reason}: {message})") + + msg = "DRPolicy validation failures:\n" + "\n".join(failures) + assert not failures, msg + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_drcluster_available(openshift_dyn_client): + """All DRClusters: phase=Available, Validated=True, not Fenced.""" + clusters = _list_resource( + openshift_dyn_client, + api_version="ramendr.openshift.io/v1alpha1", + kind="DRCluster", + ) + assert clusters, "No DRCluster objects found on hub" + + failures = [] + for c in clusters: + name = c.metadata.name + status = c.status or {} + phase = status.get("phase", "") + conditions = status.get("conditions", []) + + errors = [] + if phase != "Available": + errors.append(f"phase={phase!r} (want 'Available')") + if not _condition_status(conditions, "Validated"): + cond = _get_condition(conditions, "Validated") + if cond: + reason = cond.get("reason", "condition missing") + else: + reason = "condition missing" + errors.append(f"Validated != True ({reason})") + # Fenced=True: cluster intentionally isolated for DR + # testing; not expected in steady state + if _condition_status(conditions, "Fenced") is True: + errors.append("Fenced=True (cluster is fenced)") + + if errors: + failures.append(f"{name}: " + "; ".join(errors)) + + assert not failures, "DRCluster health failures:\n" + "\n".join(failures) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_drpc_available_and_peer_ready(openshift_dyn_client): + """ + All DRPlacementControl objects must be Available=True and PeerReady=True. + + Protected may be False/Progressing during active replication and is not + checked here — it reflects an in-flight state, not an error. + """ + drpcs = _list_resource( + openshift_dyn_client, + api_version="ramendr.openshift.io/v1alpha1", + kind="DRPlacementControl", + ) + assert drpcs, "No DRPlacementControl objects found on hub" + + failures = [] + for d in drpcs: + name = d.metadata.name + namespace = d.metadata.namespace + status = d.status or {} + phase = status.get("phase", "") + conditions = status.get("conditions", []) + + errors = [] + if phase not in ( + "Deployed", + "Relocating", + "FailingOver", + "Relocated", + "FailedOver", + ): + errors.append(f"phase={phase!r} (unexpected)") + if not _condition_status(conditions, "Available"): + cond = _get_condition(conditions, "Available") + if cond: + reason = cond.get("reason", "condition missing") + else: + reason = "condition missing" + errors.append(f"Available != True ({reason})") + if not _condition_status(conditions, "PeerReady"): + cond = _get_condition(conditions, "PeerReady") + if cond: + reason = cond.get("reason", "condition missing") + else: + reason = "condition missing" + errors.append(f"PeerReady != True ({reason})") + + if errors: + failures.append(f"{namespace}/{name}: " + "; ".join(errors)) + + assert not failures, "DRPC health failures:\n" + "\n".join(failures) diff --git a/variants/odf/values-odf.yaml b/variants/odf/values-odf.yaml index 2745a969..04f7990a 100644 --- a/variants/odf/values-odf.yaml +++ b/variants/odf/values-odf.yaml @@ -165,6 +165,19 @@ clusterGroup: name: cluster jqPathExpressions: - .status + # ESO 1.1+ adds schema defaults to the live ExternalSecret that the chart intentionally + # omits for ESO 1.0 compatibility. Suppress all four drifting fields at the Application + # layer (as the chart comment in hub-externalsecret.yaml instructs): + # deletionPolicy: Retain (target) + # engineVersion: v2 (target.template) + # mergePolicy: Replace (target.template) + - group: external-secrets.io + kind: ExternalSecret + name: cluster-ca-pushsecrets-import + jqPathExpressions: + - .spec.target.deletionPolicy + - .spec.target.template.engineVersion + - .spec.target.template.mergePolicy odf-dr: name: odf-dr From 6213d3eb5e189bd7e7ae07738ce7bf48bf54c61d Mon Sep 17 00:00:00 2001 From: OAharoni-RedHat Date: Wed, 9 Sep 2026 11:33:30 -0400 Subject: [PATCH 2/2] move byoc location --- Changes.md | 4 ++-- README.md | 2 +- overrides/values-cluster-names.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Changes.md b/Changes.md index 5906e048..f6d85f53 100644 --- a/Changes.md +++ b/Changes.md @@ -3,8 +3,8 @@ v1.3 - September 2026 * Drop the ODF-MCO `drpartner-s4` variant and rename `rhdr-s4` to `drpartner-s4`. -* `drpartner-minimal` uses the same preview RHDR catalog/IDMS/subscription and - defaults to `byoc: true`. The `odf` variant is unchanged. +* `drpartner-minimal` uses the same preview RHDR catalog/IDMS/subscription. +* Patterns are now defaulted to `byoc: true`. v1.3 - August 2026 diff --git a/README.md b/README.md index b637ecdd..fc011f81 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Spoke BOMs nest under the install variant as | Variant | Select | Purpose | |---------|--------|---------| -| `odf` (default) | `main.variant: odf` | Baseline: full ODF Regional DR + Virtualization | +| `odf` (default) | `main.variant: odf` | Baseline: full ODF Regional DR + Virtualization; `byoc: true` | | `drpartner-s4` | `main.variant: drpartner-s4` | Partner CSI + hub S4: OADP, OCP-V, preview RHDR; infrastructure DRClusters + `2m-novm` DRPolicy (no `2m-vm`/DRPC/VMs); Submariner disabled; `byoc: true` | | `drpartner-minimal` | `main.variant: drpartner-minimal` | Partner CSI without S4, Submariner, or DRCluster sync/validation: OADP, OCP-V, preview RHDR; Hive/BYOC only (`byoc: true`) | diff --git a/overrides/values-cluster-names.yaml b/overrides/values-cluster-names.yaml index 239d6424..756cfab4 100644 --- a/overrides/values-cluster-names.yaml +++ b/overrides/values-cluster-names.yaml @@ -34,4 +34,4 @@ drpc: # ./pattern.sh make install-byoc. # That target will run a validation play to ensure the cluster # setup will work with ramen patterns -byoc: true \ No newline at end of file +byoc: true