From 54ce9d540b24746d8b68f7031192f5a78ec673d2 Mon Sep 17 00:00:00 2001 From: Manuel Lorenzo Date: Tue, 16 Jun 2026 13:03:26 +0200 Subject: [PATCH 1/2] mbp-1126: Network segmentation using UDN Signed-off-by: Manuel Lorenzo --- .../templates/postgresql-statefulset.yaml | 4 + charts/qtodo/templates/app-deployment.yaml | 3 + .../templates/udn-admin-network-policy.yaml | 118 +++++++++ .../udn-network-attachment-definition.yaml | 23 ++ .../templates/udn-user-defined-network.yaml | 32 +++ charts/qtodo/values.yaml | 53 ++++ docs/SYNC-WAVE-INVENTORY.md | 8 +- docs/user-defined-networks.md | 236 ++++++++++++++++++ scripts/features/features.yaml | 3 + scripts/features/udn.yaml | 9 + 10 files changed, 487 insertions(+), 2 deletions(-) create mode 100644 charts/qtodo/templates/udn-admin-network-policy.yaml create mode 100644 charts/qtodo/templates/udn-network-attachment-definition.yaml create mode 100644 charts/qtodo/templates/udn-user-defined-network.yaml create mode 100644 docs/user-defined-networks.md create mode 100644 scripts/features/udn.yaml diff --git a/charts/qtodo-db/templates/postgresql-statefulset.yaml b/charts/qtodo-db/templates/postgresql-statefulset.yaml index c03f36b5..edd0b0ed 100644 --- a/charts/qtodo-db/templates/postgresql-statefulset.yaml +++ b/charts/qtodo-db/templates/postgresql-statefulset.yaml @@ -20,6 +20,10 @@ spec: serviceName: qtodo-db template: metadata: +{{- if .Values.app.udn.enabled }} + annotations: + k8s.v1.cni.cncf.io/networks: {{ .Release.Namespace }}/{{ .Values.app.udn.nadName }} +{{- end }} labels: app: qtodo-db spec: diff --git a/charts/qtodo/templates/app-deployment.yaml b/charts/qtodo/templates/app-deployment.yaml index 6600c34b..83d4fdf8 100644 --- a/charts/qtodo/templates/app-deployment.yaml +++ b/charts/qtodo/templates/app-deployment.yaml @@ -21,6 +21,9 @@ spec: {{- if .Values.app.spire.enabled }} checksum/app-spiffe-helper-config: {{ include (print $.Template.BasePath "/spiffe-helper-config.yaml") . | sha256sum }} checksum/app-spiffe-vault-client-config: {{ include (print $.Template.BasePath "/spiffe-vault-client-config.yaml") . | sha256sum }} +{{- end }} +{{- if .Values.app.udn.enabled }} + k8s.v1.cni.cncf.io/networks: {{ .Release.Namespace }}/{{ .Values.app.udn.nadName }} {{- end }} labels: app: qtodo diff --git a/charts/qtodo/templates/udn-admin-network-policy.yaml b/charts/qtodo/templates/udn-admin-network-policy.yaml new file mode 100644 index 00000000..71229b27 --- /dev/null +++ b/charts/qtodo/templates/udn-admin-network-policy.yaml @@ -0,0 +1,118 @@ +{{- if and .Values.app.udn.enabled .Values.app.udn.networkPolicy.enabled }} +# AdminNetworkPolicy for qtodo UDN +# Controls traffic on the secondary (UDN) interface with explicit allow-lists +# Primary network policies (cluster network) are in qtodo-network-policy.yaml +apiVersion: policy.networking.k8s.io/v1alpha1 +kind: AdminNetworkPolicy +metadata: + annotations: + argocd.argoproj.io/sync-wave: '37' + name: qtodo-udn-policy +spec: + priority: 50 + subject: + namespaces: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + ingress: + {{- if .Values.app.udn.networkPolicy.ingress.router.enabled }} + # Allow ingress from OpenShift router on primary network (not UDN) + # Router traffic comes through the cluster network interface + - name: allow-router-ingress + action: Allow + from: + - namespaces: + matchLabels: + policy-group.network.openshift.io/ingress: "" + ports: + - portNumber: + protocol: TCP + port: {{ .Values.app.udn.networkPolicy.ingress.router.port }} + {{- end }} + {{- if .Values.app.udn.networkPolicy.egress.postgresql.enabled }} + # Allow ingress to PostgreSQL from qtodo pods within same namespace on UDN + - name: allow-postgresql-ingress + action: Allow + from: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + matchLabels: + app: qtodo + ports: + - portNumber: + protocol: TCP + port: {{ .Values.app.udn.networkPolicy.egress.postgresql.port }} + {{- end }} + # Deny all other ingress by default on UDN + - name: deny-all-ingress + action: Deny + from: + - namespaces: {} + egress: + {{- if .Values.app.udn.networkPolicy.egress.dns.enabled }} + # Allow DNS resolution via CoreDNS + - name: allow-dns + action: Allow + to: + - namespaces: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.app.udn.networkPolicy.egress.dns.namespace }} + ports: + - portNumber: + protocol: UDP + port: {{ .Values.app.udn.networkPolicy.egress.dns.port }} + - portNumber: + protocol: TCP + port: {{ .Values.app.udn.networkPolicy.egress.dns.port }} + {{- end }} + {{- if .Values.app.udn.networkPolicy.egress.postgresql.enabled }} + # Allow PostgreSQL access within same namespace on UDN + - name: allow-postgresql + action: Allow + to: + - pods: + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + matchLabels: + app: qtodo-db + ports: + - portNumber: + protocol: TCP + port: {{ .Values.app.udn.networkPolicy.egress.postgresql.port }} + {{- end }} + {{- if .Values.app.udn.networkPolicy.egress.vault.enabled }} + # Allow Vault API access (SPIFFE JWT auth) via cluster network + - name: allow-vault + action: Allow + to: + - namespaces: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.app.udn.networkPolicy.egress.vault.namespace }} + ports: + - portNumber: + protocol: TCP + port: {{ .Values.app.udn.networkPolicy.egress.vault.port }} + {{- end }} + {{- if .Values.app.udn.networkPolicy.egress.https.enabled }} + # Allow OIDCs (HTTPS connections) via cluster network (external route) + - name: allow-https + action: Allow + to: + - networks: + - 0.0.0.0/0 # External route, cannot match by namespace + ports: + - portNumber: + protocol: TCP + port: {{ .Values.app.udn.networkPolicy.egress.https.port }} + {{- end }} + # Deny all other egress by default on UDN + - name: deny-all-egress + action: Deny + to: + - namespaces: {} +{{- end }} diff --git a/charts/qtodo/templates/udn-network-attachment-definition.yaml b/charts/qtodo/templates/udn-network-attachment-definition.yaml new file mode 100644 index 00000000..321a3621 --- /dev/null +++ b/charts/qtodo/templates/udn-network-attachment-definition.yaml @@ -0,0 +1,23 @@ +{{- if .Values.app.udn.enabled }} +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + annotations: + argocd.argoproj.io/sync-wave: '36' + description: "Network attachment for qtodo UDN isolation" + name: {{ .Values.app.udn.nadName }} + namespace: {{ .Release.Namespace }} +spec: + config: | + { + "cniVersion": "0.4.0", + "type": "ovn-k8s-cni-overlay", + "name": "{{ .Values.app.udn.name }}", + "topology": "{{ lower .Values.app.udn.topology }}", + "netAttachDefName": "{{ .Release.Namespace }}/{{ .Values.app.udn.nadName }}", + {{- if eq .Values.app.udn.topology "Layer2" }} + "subnets": "{{ .Values.app.udn.subnet }}", + {{- end }} + "mtu": {{ .Values.app.udn.mtu }} + } +{{- end }} diff --git a/charts/qtodo/templates/udn-user-defined-network.yaml b/charts/qtodo/templates/udn-user-defined-network.yaml new file mode 100644 index 00000000..40ec33d8 --- /dev/null +++ b/charts/qtodo/templates/udn-user-defined-network.yaml @@ -0,0 +1,32 @@ +{{- if .Values.app.udn.enabled }} +apiVersion: k8s.ovn.org/v1 +kind: UserDefinedNetwork +metadata: + annotations: + argocd.argoproj.io/sync-wave: '35' + name: {{ .Values.app.udn.name }} + namespace: {{ .Release.Namespace }} +spec: + topology: {{ .Values.app.udn.topology }} + {{- if eq .Values.app.udn.topology "Layer2" }} + layer2: + role: Primary + subnets: + - {{ .Values.app.udn.subnet }} + {{- if .Values.app.udn.mtu }} + mtu: {{ .Values.app.udn.mtu }} + {{- end }} + {{- else if eq .Values.app.udn.topology "Layer3" }} + layer3: + role: Primary + subnets: + - {{ .Values.app.udn.subnet }} + {{- if .Values.app.udn.joinSubnet }} + joinSubnets: + - {{ .Values.app.udn.joinSubnet }} + {{- end }} + {{- if .Values.app.udn.mtu }} + mtu: {{ .Values.app.udn.mtu }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/qtodo/values.yaml b/charts/qtodo/values.yaml index 6d7ebded..f820cdbe 100644 --- a/charts/qtodo/values.yaml +++ b/charts/qtodo/values.yaml @@ -114,6 +114,59 @@ app: # QTodo truststore password path (app-level isolation) vaultPath: "secret/data/apps/qtodo/qtodo-truststore" + # User-Defined Network (UDN) configuration for network isolation + # Provides layer 2/3 network segmentation following Zero Trust principles + udn: + enabled: false + # Network name + name: qtodo-isolated-network + # NetworkAttachmentDefinition name + nadName: qtodo-udn-nad + # Topology: Layer2 or Layer3 + # Layer2: Same subnet, pods can communicate directly + # Layer3: Different subnets per node, requires routing + topology: Layer2 + # CIDR for the UDN subnet (Layer2 only) + subnet: "10.100.0.0/16" + # Join subnet (for Layer3, optional) + # joinSubnet: "100.64.0.0/16" + # MTU for the network (default: 1400 to avoid fragmentation) + mtu: 1400 + # IPAM configuration + ipam: + type: static + # Network Policy for UDN + # UDN requires policies on both primary (cluster network) and secondary (UDN) interfaces + networkPolicy: + # Enable network policies on the UDN + enabled: true + # Allowed egress destinations from qtodo pods on the UDN + egress: + # DNS resolution + dns: + enabled: true + port: 5353 + namespace: openshift-dns + # PostgreSQL database + postgresql: + enabled: true + port: 5432 + # Vault for secrets (via primary network, not UDN) + vault: + enabled: true + port: 8200 + namespace: vault + # Allow HTTPS connections to any destination, used for OIDCs (via primary network for external route) + https: + enabled: true + port: 443 + # Allowed ingress sources to qtodo pods on the UDN + ingress: + # OpenShift router (via primary network, not UDN) + router: + enabled: true + port: 8080 + # PostgreSQL database configuration postgresql: name: qtodo-db diff --git a/docs/SYNC-WAVE-INVENTORY.md b/docs/SYNC-WAVE-INVENTORY.md index 45c43613..4625cab6 100644 --- a/docs/SYNC-WAVE-INVENTORY.md +++ b/docs/SYNC-WAVE-INVENTORY.md @@ -48,18 +48,20 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | 34 | └ rhtpa-operator | chart | oidc-cli-secret | | 34 | └ noobaa-mcg | chart | bucket-class | | 35 | rh-keycloak | **App** | | +| 35 | └ qtodo | chart | udn-user-defined-network (UserDefinedNetwork CR for network isolation) | | 36 | noobaa-mcg | **App** | | | 36 | └ rhtpa-operator | chart | postgresql-serviceaccount, postgresql-external-secret, object-bucket-claim | | 36 | └ keycloak | chart | keycloak.yaml (Keycloak CR) | | 36 | └ quay-registry | chart | object-bucket-claim | | 36 | └ acs-central | chart | admin-password-secret, central-htpasswd-external-secret, keycloak-client-secret-external-secret | -| 36 | └ qtodo | chart | truststore-secret-external-secret, registry-external-secret | | 36 | └ qtodo-db | chart | postgresql-external-secret | +| 36 | └ qtodo | chart | udn-network-attachment-definition (NAD), truststore-secret-external-secret, registry-external-secret | | 38+0 | └ qtodo | chart | registry-seed SA, ClusterRole, ClusterRoleBinding | | 38+5 | └ qtodo | chart (hook) | registry-seed-image (Sync hook Job -- mirrors upstream image to configured registry) | | 37 | qtodo-db | **App** | PostgreSQL for qtodo (before qtodo app) | | 37 | └ quay-registry | chart | quay-s3-setup-serviceaccount (5 resources) | | 37 | └ acs-central | chart | create-htpasswd-field (Job) | +| 37 | └ qtodo | chart | udn-admin-network-policy (AdminNetworkPolicy for UDN traffic control) | | 38 | qtodo | **App** | | | 38 | └ quay-registry | chart | quay-config-bundle-secret | | 39 | └ rhtpa-operator | chart | s3-credentials-secret | @@ -284,9 +286,11 @@ Charts marked **(external)** have been externalized to standalone repositories m | --- | ---: | ---: | | registry-seed-job.yaml (SA, ClusterRole, ClusterRoleBinding) | --- | 0 | | registry-seed-job.yaml (Sync hook Job) | --- | 5 | +| udn-user-defined-network.yaml | --- | 35 | +| udn-network-attachment-definition.yaml | --- | 36 | | truststore-secret-external-secret.yaml | 5 | 36 | | registry-external-secret.yaml | --- | 36 | -| postgresql-external-secret.yaml (SPIFFE-off only) | 5 | 36 | +| udn-admin-network-policy.yaml (AdminNetworkPolicy) | --- | 37 | | qtodo-truststore-config.yaml | 10 | 41 | | app-deployment.yaml | 20 | 51 | | app-service.yaml | 20 | 51 | diff --git a/docs/user-defined-networks.md b/docs/user-defined-networks.md new file mode 100644 index 00000000..f6681791 --- /dev/null +++ b/docs/user-defined-networks.md @@ -0,0 +1,236 @@ +# User-Defined Networks (UDN) for Zero Trust Network Isolation + +## Overview + +User-Defined Networks (UDN) provide layer 2/3 network isolation for workloads in OpenShift, separate from the default cluster network. This feature implements Zero Trust network segmentation principles by creating an isolated network for the qtodo application, restricting communication to only necessary services. + +## Architecture + +### Network Topology + +The UDN implementation creates a dedicated isolated network for qtodo workloads: + +```text +┌─────────────────────────────────────────────────────┐ +│ Cluster Network │ +│ ┌────────────┐ ┌─────────┐ ┌──────────┐ │ +│ │ Router │───▶│ qtodo │───▶│ Vault │ │ +│ │ (Ingress) │ │ (eth0) │ │ (8200) │ │ +│ └────────────┘ └────┬────┘ └──────────┘ │ +│ │ │ +│ │ UDN Attachment │ +└─────────────────────────┼───────────────────────────┘ + │ + ┌─────▼──────┐ + │ UDN │ + │ (net1/ │ + │ Layer2) │ + └─────┬──────┘ + │ + ┌─────────┴────────┐ + │ │ + ┌─────▼──────┐ ┌────▼─────┐ + │ qtodo pod │ │ qtodo-db │ + │ (isolated) │──────│ (5432) │ + └────────────┘ └──────────┘ + │ + └─────────▶ DNS (5353) via cluster network +``` + +### Dual Network Interfaces + +When UDN is enabled, qtodo pods have two network interfaces: + +1. **eth0 (Primary - Cluster Network)** + - Ingress from OpenShift Router (port 8080) + - Egress to Vault (SPIFFE auth, port 8200) + - Egress to OIDCs (OIDC back-channel, port 443) + - DNS resolution (CoreDNS, port 5353) + +2. **net1 (Secondary - UDN)** + - PostgreSQL communication (qtodo ↔ qtodo-db, port 5432) + - Isolated from other cluster workloads + - Layer 2 topology (same subnet across nodes) + +## Security Benefits + +1. **Network Segmentation**: qtodo workloads are isolated from arbitrary cluster traffic +2. **Explicit Allow-Lists**: AdminNetworkPolicy enforces allow-only-required communication +3. **Defense in Depth**: Combines with existing NetworkPolicy for dual-layer protection +4. **Blast Radius Reduction**: Compromise of qtodo cannot pivot to unrelated services +5. **Compliance**: Supports Zero Trust architecture mandates (NIST 800-207, NIS2, ISO 27001:2022) + +## Components + +UDN is integrated into the qtodo Helm chart (`charts/qtodo`). When enabled, the following resources are created: + +### UserDefinedNetwork CR + +Template: `charts/qtodo/templates/udn-user-defined-network.yaml` + +Creates the isolated network with Layer2 topology: + +- Subnet: `10.100.0.0/16` +- MTU: 1400 (avoids fragmentation) +- IPAM: Persistent IP assignment +- Sync-wave: 35 (before NAD) + +### NetworkAttachmentDefinition + +Template: `charts/qtodo/templates/udn-network-attachment-definition.yaml` + +Defines how pods attach to the UDN: + +- CNI type: `ovn-k8s-cni-overlay` +- References the UserDefinedNetwork +- Used via pod annotation `k8s.v1.cni.cncf.io/networks` +- Sync-wave: 36 (before policies) + +### AdminNetworkPolicy + +Template: `charts/qtodo/templates/udn-admin-network-policy.yaml` + +Explicit allow-list for UDN traffic: + +- **Ingress**: + - OpenShift router (port 8080) + - qtodo pods to qtodo-db (port 5432) +- **Egress**: DNS, PostgreSQL, Vault, Keycloak (HTTPS connections) +- Priority: 50 (higher = processed first) +- Sync-wave: 37 (before qtodo app) + +## Enabling UDN + +### Option 1: Feature Variant Generator (Recommended) + +```bash +python3 scripts/gen-feature-variants.py \ + --features udn \ + --base values-hub.yaml + +# Apply the variant +cp /tmp/values-hub-udn.yaml values-hub.yaml +./pattern.sh make install +``` + +### Option 2: Manual Configuration + +1. **Enable UDN in the qtodo application** in `values-hub.yaml`: + + ```yaml + clusterGroup: + applications: + qtodo: + # ... existing config ... + overrides: + # ... existing overrides ... + - name: app.udn.enabled + value: "true" + ``` + +2. **Deploy**: + + ```bash + ./pattern.sh make install + ``` + +## Verification + +### 1. Check UDN Resources + +```bash +# UserDefinedNetwork +oc get userdefinednetwork -n qtodo +NAME AGE +qtodo-isolated-network 5m + +# NetworkAttachmentDefinition +oc get network-attachment-definitions -n qtodo +NAME AGE +qtodo-udn-nad 5m +``` + +### 2. Verify Network Policies + +```bash +# AdminNetworkPolicy +oc get adminnetworkpolicy +NAME PRIORITY AGE +qtodo-udn-policy 50 5m +``` + +### 3. Test Connectivity + +```bash +# DNS resolution (should work via eth0) +oc exec -n qtodo deploy/qtodo -c qtodo -- getent hosts qtodo-db + +# PostgreSQL connectivity (should work via net1) +oc exec -n qtodo deploy/qtodo -c qtodo -- timeout 5 bash -c '/dev/null' && echo "OK" + +# Vault API (should work via eth0) +oc exec -n qtodo deploy/qtodo -c qtodo -- curl -sk https://vault.vault.svc:8200/v1/sys/health +``` + +### 4. Verify qtodo Application + +```bash +# Get the route +QTODO_URL=$(oc get route -n qtodo qtodo -o jsonpath='{.spec.host}') + +# Access the application +curl https://$QTODO_URL +``` + +## Configuration Options + +UDN is configured via the `app.udn` section in `charts/qtodo/values.yaml`: + +| Parameter | Description | Default | +| -------------------------------- | -------------------------------- | ------------------------ | +| `app.udn.enabled` | Enable UDN | `false` | +| `app.udn.name` | UserDefinedNetwork name | `qtodo-isolated-network` | +| `app.udn.nadName` | NetworkAttachmentDefinition name | `qtodo-udn-nad` | +| `app.udn.topology` | Network topology (Layer2/Layer3) | `Layer2` | +| `app.udn.subnet` | CIDR for UDN | `10.100.0.0/16` | +| `app.udn.mtu` | MTU for the network | `1400` | +| `app.udn.networkPolicy.enabled` | Enable AdminNetworkPolicy | `true` | + +### Layer3 Topology + +For larger deployments, Layer3 provides better scalability. Override in `values-hub.yaml`: + +```yaml +clusterGroup: + applications: + qtodo: + overrides: + - name: app.udn.enabled + value: "true" + - name: app.udn.topology + value: "Layer3" + - name: app.udn.joinSubnet + value: "100.64.0.0/16" +``` + +## Security Considerations + +### Defense in Depth + +UDN complements, but does not replace, other security controls: + +- **NetworkPolicy**: Still applied on the cluster network (eth0) +- **Service Mesh**: mTLS can layer on top of UDN +- **ACS Policies**: Runtime enforcement still active + +### Attack Surface + +- UDN pods are still reachable via cluster network (eth0) for ingress/egress to external services +- AdminNetworkPolicy must be correctly configured to avoid bypasses +- Pods with `CAP_NET_ADMIN` could potentially manipulate interfaces +- For integration with IDPs (_Keycloak_, _EntraID_), HTTPS connections to any destination are enabled. In a more secure environment, this rule should be more restrictive and only allow access to specific destinations. + +## References + +- [OpenShift UDN Documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/latest/html/multiple_networks/understanding-multiple-networks) +- [OVN-Kubernetes User-Defined Networks](https://github.com/ovn-kubernetes/ovn-kubernetes/blob/master/docs/features/user-defined-networks/user-defined-networks.md) diff --git a/scripts/features/features.yaml b/scripts/features/features.yaml index 5a324cde..cb4f80be 100644 --- a/scripts/features/features.yaml +++ b/scripts/features/features.yaml @@ -48,6 +48,9 @@ features: entra-id-qtodo: description: "Azure Entra ID for qtodo authentication (no supply-chain)" + + udn: + description: "User-Defined Network isolation for qtodo" depends_on: [] # Registry options (only used with supply-chain feature) diff --git a/scripts/features/udn.yaml b/scripts/features/udn.yaml new file mode 100644 index 00000000..71c8d383 --- /dev/null +++ b/scripts/features/udn.yaml @@ -0,0 +1,9 @@ +# User-Defined Network isolation for qtodo application +# Provides layer 2/3 network segmentation following Zero Trust principles +clusterGroup: + # Merge UDN configuration into existing qtodo application + merge_into_applications: + qtodo: + overrides: + - name: app.udn.enabled + value: "true" From 70d6aadcd17311530a5df60060e51aa007948d1b Mon Sep 17 00:00:00 2001 From: Manuel Lorenzo Date: Wed, 9 Sep 2026 15:39:13 +0200 Subject: [PATCH 2/2] Refactor UDN segmentation for the split qtodo/qtodo-db namespaces Signed-off-by: Manuel Lorenzo --- charts/qtodo-db/templates/_helpers.tpl | 39 +++ .../templates/postgresql-statefulset.yaml | 5 +- .../templates/qtodo-db-network-policy.yaml | 10 +- .../udn-cluster-user-defined-network.yaml | 38 ++ .../templates/udn-multi-network-policy.yaml | 48 +++ charts/qtodo-db/values.yaml | 24 +- charts/qtodo/templates/_helpers.tpl | 31 ++ charts/qtodo/templates/app-deployment.yaml | 6 +- .../qtodo/templates/qtodo-network-policy.yaml | 7 +- .../templates/udn-admin-network-policy.yaml | 118 ------- .../udn-enable-multi-network-policy-job.yaml | 65 ++++ .../udn-enable-multi-network-policy-rbac.yaml | 43 +++ .../templates/udn-multi-network-policy.yaml | 48 +++ .../udn-network-attachment-definition.yaml | 23 -- .../templates/udn-user-defined-network.yaml | 32 -- charts/qtodo/values.yaml | 70 ++-- docs/README.md | 1 + docs/SYNC-WAVE-INVENTORY.md | 17 +- docs/USER-JOURNEYS.md | 2 +- docs/multi-tier.md | 6 + docs/user-defined-networks.md | 326 ++++++++++-------- scripts/features/features.yaml | 3 +- scripts/features/udn.yaml | 10 +- scripts/gen-feature-variants.md | 17 + values-hub.yaml | 8 + 25 files changed, 615 insertions(+), 382 deletions(-) create mode 100644 charts/qtodo-db/templates/udn-cluster-user-defined-network.yaml create mode 100644 charts/qtodo-db/templates/udn-multi-network-policy.yaml delete mode 100644 charts/qtodo/templates/udn-admin-network-policy.yaml create mode 100644 charts/qtodo/templates/udn-enable-multi-network-policy-job.yaml create mode 100644 charts/qtodo/templates/udn-enable-multi-network-policy-rbac.yaml create mode 100644 charts/qtodo/templates/udn-multi-network-policy.yaml delete mode 100644 charts/qtodo/templates/udn-network-attachment-definition.yaml delete mode 100644 charts/qtodo/templates/udn-user-defined-network.yaml diff --git a/charts/qtodo-db/templates/_helpers.tpl b/charts/qtodo-db/templates/_helpers.tpl index 865b37ce..0f59bac3 100644 --- a/charts/qtodo-db/templates/_helpers.tpl +++ b/charts/qtodo-db/templates/_helpers.tpl @@ -9,3 +9,42 @@ Create the image path for the passed in image field. {{- printf "%s:%s" $name (tpl .value.version .context) -}} {{- end -}} {{- end -}} + +{{/* +True when the shared secondary UDN is enabled (bool or string "true"). +*/}} +{{- define "qtodo-db.udn.enabled" -}} +{{- if eq (.Values.udn.enabled | default false | toString) "true" -}} +true +{{- end -}} +{{- end -}} + +{{/* +IPv4 prefix length from udn.subnet (e.g. 10.100.0.0/16 -> 16). +*/}} +{{- define "qtodo-db.udn.prefixlen" -}} +{{- regexReplaceAll "^[^/]+/" .Values.udn.subnet "" -}} +{{- end -}} + +{{/* +Multus annotation attaching the postgres pod to the CUDN-managed NAD. +Uses simple / format — static IP is driven by the +k8s.ovn.org/pod-networks pre-annotation, not by the Multus ips field +(which OVN-K does not support for secondary Layer2 UDN networks). +*/}} +{{- define "qtodo-db.udn.networksAnnotation" -}} +{{ .Release.Namespace }}/{{ .Values.udn.name }} +{{- end -}} + +{{/* +OVN-K pod-networks pre-annotation that requests a static IP on the +secondary UDN. The key format is / — matching the +key OVN-K writes when it allocates dynamically. OVN-K reads this before +the CNI call and honours the specified ip_addresses instead of allocating +a new one from the pool. +*/}} +{{- define "qtodo-db.udn.ovnPodNetworks" -}} +{{- $ip := printf "%s/%s" .Values.udn.dbIP (include "qtodo-db.udn.prefixlen" .) -}} +{{- $key := printf "%s/%s" .Release.Namespace .Values.udn.name -}} +{"{{ $key }}":{"ip_addresses":["{{ $ip }}"],"ip_address":"{{ $ip }}","role":"secondary"}} +{{- end -}} diff --git a/charts/qtodo-db/templates/postgresql-statefulset.yaml b/charts/qtodo-db/templates/postgresql-statefulset.yaml index edd0b0ed..a4c1aa0e 100644 --- a/charts/qtodo-db/templates/postgresql-statefulset.yaml +++ b/charts/qtodo-db/templates/postgresql-statefulset.yaml @@ -20,9 +20,10 @@ spec: serviceName: qtodo-db template: metadata: -{{- if .Values.app.udn.enabled }} +{{- if include "qtodo-db.udn.enabled" . }} annotations: - k8s.v1.cni.cncf.io/networks: {{ .Release.Namespace }}/{{ .Values.app.udn.nadName }} + k8s.v1.cni.cncf.io/networks: {{ include "qtodo-db.udn.networksAnnotation" . | quote }} + k8s.ovn.org/pod-networks: {{ include "qtodo-db.udn.ovnPodNetworks" . | quote }} {{- end }} labels: app: qtodo-db diff --git a/charts/qtodo-db/templates/qtodo-db-network-policy.yaml b/charts/qtodo-db/templates/qtodo-db-network-policy.yaml index 18eeef70..19a5c767 100644 --- a/charts/qtodo-db/templates/qtodo-db-network-policy.yaml +++ b/charts/qtodo-db/templates/qtodo-db-network-policy.yaml @@ -10,8 +10,9 @@ spec: policyTypes: - Ingress - Egress +{{- if eq (include "qtodo-db.udn.enabled" .) "" }} ingress: - # PostgreSQL — only from qtodo app pods in the qtodo namespace + # PostgreSQL — only from qtodo app pods in the qtodo namespace (cluster network) - ports: - protocol: TCP port: {{ .Values.postgresql.port }} @@ -22,8 +23,13 @@ spec: podSelector: matchLabels: app: {{ .Values.postgresql.allowedClientApp }} +{{- end }} + # When UDN is enabled there is no ingress allow-list: default-deny plus + # policyTypes: Ingress blocks PostgreSQL on the cluster network. Clients + # must use the shared secondary UDN (see udn-multi-network-policy.yaml). egress: - # DNS resolution via CoreDNS — OCP uses port 5353 (not 53) + # DNS resolution via CoreDNS — OCP uses port 5353 (not 53). This is the + # only egress qtodo-db is permitted on the cluster network. - ports: - protocol: UDP port: 5353 diff --git a/charts/qtodo-db/templates/udn-cluster-user-defined-network.yaml b/charts/qtodo-db/templates/udn-cluster-user-defined-network.yaml new file mode 100644 index 00000000..f5d2e6d5 --- /dev/null +++ b/charts/qtodo-db/templates/udn-cluster-user-defined-network.yaml @@ -0,0 +1,38 @@ +{{- if include "qtodo-db.udn.enabled" . }} +{{- if ne .Values.udn.topology "Layer2" }} +{{- fail "qtodo-db UDN supports Layer2 topology only (shared secondary ClusterUserDefinedNetwork)" }} +{{- end }} +{{- if ne .Values.udn.role "Secondary" }} +{{- fail "qtodo-db UDN must use role Secondary so qtodo can keep the cluster network as its primary interface" }} +{{- end }} +# Cluster-scoped Layer2 secondary UDN shared by the qtodo and qtodo-db +# namespaces. OVN-Kubernetes creates a NetworkAttachmentDefinition of the +# same name in each selected namespace; pods opt in via the +# k8s.v1.cni.cncf.io/networks annotation. reservedSubnets is not used here +# because it is only valid for Primary networks. +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + annotations: + argocd.argoproj.io/sync-wave: "35" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + name: {{ .Values.udn.name }} +spec: + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + {{- range .Values.udn.namespaces }} + - {{ . | quote }} + {{- end }} + network: + topology: {{ .Values.udn.topology }} + layer2: + role: {{ .Values.udn.role }} + subnets: + - {{ .Values.udn.subnet }} + {{- if .Values.udn.mtu }} + mtu: {{ .Values.udn.mtu }} + {{- end }} +{{- end }} diff --git a/charts/qtodo-db/templates/udn-multi-network-policy.yaml b/charts/qtodo-db/templates/udn-multi-network-policy.yaml new file mode 100644 index 00000000..0bfef17e --- /dev/null +++ b/charts/qtodo-db/templates/udn-multi-network-policy.yaml @@ -0,0 +1,48 @@ +{{- if and (include "qtodo-db.udn.enabled" .) .Values.udn.networkPolicy.enabled }} +# Default deny on the secondary UDN, then allow PostgreSQL only from qtodo. +# MultiNetworkPolicy applies to the CUDN-managed NAD, not the cluster network. +# Requires spec.useMultiNetworkPolicy: true on network.operator.openshift.io/cluster. +apiVersion: k8s.cni.cncf.io/v1beta1 +kind: MultiNetworkPolicy +metadata: + annotations: + argocd.argoproj.io/sync-wave: "37" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + k8s.v1.cni.cncf.io/policy-for: {{ .Release.Namespace }}/{{ .Values.udn.name }} + name: default-deny-udn + namespace: {{ .Release.Namespace }} +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: [] + egress: [] +--- +apiVersion: k8s.cni.cncf.io/v1beta1 +kind: MultiNetworkPolicy +metadata: + annotations: + argocd.argoproj.io/sync-wave: "37" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + k8s.v1.cni.cncf.io/policy-for: {{ .Release.Namespace }}/{{ .Values.udn.name }} + name: allow-qtodo-postgresql + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + app: qtodo-db + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.postgresql.allowedClientNamespace }} + podSelector: + matchLabels: + app: {{ .Values.postgresql.allowedClientApp }} + ports: + - protocol: TCP + port: {{ .Values.udn.networkPolicy.postgresqlPort | default .Values.postgresql.port }} +{{- end }} diff --git a/charts/qtodo-db/values.yaml b/charts/qtodo-db/values.yaml index 80ae48af..ad4753d4 100644 --- a/charts/qtodo-db/values.yaml +++ b/charts/qtodo-db/values.yaml @@ -19,6 +19,28 @@ postgresql: # qtodo DB password path (app-level isolation) passwordVaultKey: secret/data/apps/qtodo/qtodo-db - # Namespace allowed to connect to PostgreSQL + # Namespace allowed to connect to PostgreSQL on the cluster network. + # Ignored when udn.enabled is true — PostgreSQL is then reachable only + # on the shared secondary UDN (see udn.dbIP). allowedClientNamespace: qtodo allowedClientApp: qtodo + +# Shared secondary User-Defined Network (CUDN) spanning qtodo and qtodo-db. +# Keep name, subnet, and dbIP in sync with charts/qtodo values (app.udn.*). +# Kubernetes Services are not supported on secondary UDNs, so qtodo connects +# to PostgreSQL by the static UDN IP set via network annotation (udn.dbIP). +udn: + enabled: false + name: qtodo-isolated-network + topology: Layer2 + role: Secondary + subnet: "10.100.0.0/16" + mtu: 1400 + dbIP: "10.100.0.10" + namespaces: + - qtodo + - qtodo-db + networkPolicy: + # MultiNetworkPolicy on the UDN (requires CNO useMultiNetworkPolicy: true) + enabled: true + postgresqlPort: 5432 diff --git a/charts/qtodo/templates/_helpers.tpl b/charts/qtodo/templates/_helpers.tpl index 0bfd247a..62f5f6b8 100644 --- a/charts/qtodo/templates/_helpers.tpl +++ b/charts/qtodo/templates/_helpers.tpl @@ -87,4 +87,35 @@ Returns the port the application should list on {{- else -}} {{ .Values.app.insecurePort }} {{- end -}} +{{- end -}} + +{{/* +True when the shared secondary UDN is enabled (bool or string "true"). +*/}} +{{- define "qtodo.udn.enabled" -}} +{{- if eq (.Values.app.udn.enabled | default false | toString) "true" -}} +true +{{- end -}} +{{- end -}} + +{{/* +True when the CNO patch Job should run (UDN + MultiNetworkPolicy enabled, +and the job itself is not disabled because another component manages CNO). +*/}} +{{- define "qtodo.udn.multiNetworkPolicyJob.enabled" -}} +{{- if and (include "qtodo.udn.enabled" .) (eq (.Values.app.udn.networkPolicy.enabled | default false | toString) "true") (ne (.Values.app.udn.multiNetworkPolicyJob.enabled | toString) "false") -}} +true +{{- end -}} +{{- end -}} + +{{/* +PostgreSQL host: static UDN IP when UDN is enabled (secondary UDNs do not +support Kubernetes Services), otherwise the cluster-network Service DNS name. +*/}} +{{- define "qtodo.dbHost" -}} +{{- if include "qtodo.udn.enabled" . -}} +{{- .Values.app.udn.dbIP -}} +{{- else -}} +{{- .Values.postgresql.host -}} +{{- end -}} {{- end -}} \ No newline at end of file diff --git a/charts/qtodo/templates/app-deployment.yaml b/charts/qtodo/templates/app-deployment.yaml index 83d4fdf8..f0e277da 100644 --- a/charts/qtodo/templates/app-deployment.yaml +++ b/charts/qtodo/templates/app-deployment.yaml @@ -22,8 +22,8 @@ spec: checksum/app-spiffe-helper-config: {{ include (print $.Template.BasePath "/spiffe-helper-config.yaml") . | sha256sum }} checksum/app-spiffe-vault-client-config: {{ include (print $.Template.BasePath "/spiffe-vault-client-config.yaml") . | sha256sum }} {{- end }} -{{- if .Values.app.udn.enabled }} - k8s.v1.cni.cncf.io/networks: {{ .Release.Namespace }}/{{ .Values.app.udn.nadName }} +{{- if include "qtodo.udn.enabled" . }} + k8s.v1.cni.cncf.io/networks: {{ .Release.Namespace }}/{{ .Values.app.udn.name }} {{- end }} labels: app: qtodo @@ -218,7 +218,7 @@ spec: type: RuntimeDefault env: - name: QUARKUS_DATASOURCE_JDBC_URL - value: 'jdbc:postgresql://{{ .Values.postgresql.host }}:{{ .Values.postgresql.port }}/{{ .Values.postgresql.auth.database }}' + value: 'jdbc:postgresql://{{ include "qtodo.dbHost" . }}:{{ .Values.postgresql.port }}/{{ .Values.postgresql.auth.database }}' - name: QUARKUS_HTTP_HOST value: '0.0.0.0' - name: QUARKUS_HTTP_PORT diff --git a/charts/qtodo/templates/qtodo-network-policy.yaml b/charts/qtodo/templates/qtodo-network-policy.yaml index dbdc43f2..1ff5d393 100644 --- a/charts/qtodo/templates/qtodo-network-policy.yaml +++ b/charts/qtodo/templates/qtodo-network-policy.yaml @@ -33,7 +33,11 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: openshift-dns - # PostgreSQL — qtodo-db pod in the qtodo-db namespace +# When UDN is enabled, PostgreSQL is reached on the secondary UDN (net1) +# using the static IP in app.udn.dbIP. Cluster-network egress to 5432 is +# intentionally omitted so the database path cannot bypass the UDN. +{{- if eq (include "qtodo.udn.enabled" .) "" }} + # PostgreSQL — qtodo-db pod in the qtodo-db namespace (cluster network) - ports: - protocol: TCP port: {{ .Values.postgresql.port }} @@ -44,6 +48,7 @@ spec: podSelector: matchLabels: app: qtodo-db +{{- end }} # Vault API — SPIFFE JWT auth for DB credentials retrieval - ports: - protocol: TCP diff --git a/charts/qtodo/templates/udn-admin-network-policy.yaml b/charts/qtodo/templates/udn-admin-network-policy.yaml deleted file mode 100644 index 71229b27..00000000 --- a/charts/qtodo/templates/udn-admin-network-policy.yaml +++ /dev/null @@ -1,118 +0,0 @@ -{{- if and .Values.app.udn.enabled .Values.app.udn.networkPolicy.enabled }} -# AdminNetworkPolicy for qtodo UDN -# Controls traffic on the secondary (UDN) interface with explicit allow-lists -# Primary network policies (cluster network) are in qtodo-network-policy.yaml -apiVersion: policy.networking.k8s.io/v1alpha1 -kind: AdminNetworkPolicy -metadata: - annotations: - argocd.argoproj.io/sync-wave: '37' - name: qtodo-udn-policy -spec: - priority: 50 - subject: - namespaces: - matchLabels: - kubernetes.io/metadata.name: {{ .Release.Namespace }} - ingress: - {{- if .Values.app.udn.networkPolicy.ingress.router.enabled }} - # Allow ingress from OpenShift router on primary network (not UDN) - # Router traffic comes through the cluster network interface - - name: allow-router-ingress - action: Allow - from: - - namespaces: - matchLabels: - policy-group.network.openshift.io/ingress: "" - ports: - - portNumber: - protocol: TCP - port: {{ .Values.app.udn.networkPolicy.ingress.router.port }} - {{- end }} - {{- if .Values.app.udn.networkPolicy.egress.postgresql.enabled }} - # Allow ingress to PostgreSQL from qtodo pods within same namespace on UDN - - name: allow-postgresql-ingress - action: Allow - from: - - pods: - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: {{ .Release.Namespace }} - podSelector: - matchLabels: - app: qtodo - ports: - - portNumber: - protocol: TCP - port: {{ .Values.app.udn.networkPolicy.egress.postgresql.port }} - {{- end }} - # Deny all other ingress by default on UDN - - name: deny-all-ingress - action: Deny - from: - - namespaces: {} - egress: - {{- if .Values.app.udn.networkPolicy.egress.dns.enabled }} - # Allow DNS resolution via CoreDNS - - name: allow-dns - action: Allow - to: - - namespaces: - matchLabels: - kubernetes.io/metadata.name: {{ .Values.app.udn.networkPolicy.egress.dns.namespace }} - ports: - - portNumber: - protocol: UDP - port: {{ .Values.app.udn.networkPolicy.egress.dns.port }} - - portNumber: - protocol: TCP - port: {{ .Values.app.udn.networkPolicy.egress.dns.port }} - {{- end }} - {{- if .Values.app.udn.networkPolicy.egress.postgresql.enabled }} - # Allow PostgreSQL access within same namespace on UDN - - name: allow-postgresql - action: Allow - to: - - pods: - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: {{ .Release.Namespace }} - podSelector: - matchLabels: - app: qtodo-db - ports: - - portNumber: - protocol: TCP - port: {{ .Values.app.udn.networkPolicy.egress.postgresql.port }} - {{- end }} - {{- if .Values.app.udn.networkPolicy.egress.vault.enabled }} - # Allow Vault API access (SPIFFE JWT auth) via cluster network - - name: allow-vault - action: Allow - to: - - namespaces: - matchLabels: - kubernetes.io/metadata.name: {{ .Values.app.udn.networkPolicy.egress.vault.namespace }} - ports: - - portNumber: - protocol: TCP - port: {{ .Values.app.udn.networkPolicy.egress.vault.port }} - {{- end }} - {{- if .Values.app.udn.networkPolicy.egress.https.enabled }} - # Allow OIDCs (HTTPS connections) via cluster network (external route) - - name: allow-https - action: Allow - to: - - networks: - - 0.0.0.0/0 # External route, cannot match by namespace - ports: - - portNumber: - protocol: TCP - port: {{ .Values.app.udn.networkPolicy.egress.https.port }} - {{- end }} - # Deny all other egress by default on UDN - - name: deny-all-egress - action: Deny - to: - - namespaces: {} -{{- end }} diff --git a/charts/qtodo/templates/udn-enable-multi-network-policy-job.yaml b/charts/qtodo/templates/udn-enable-multi-network-policy-job.yaml new file mode 100644 index 00000000..f286ed8f --- /dev/null +++ b/charts/qtodo/templates/udn-enable-multi-network-policy-job.yaml @@ -0,0 +1,65 @@ +{{- if include "qtodo.udn.multiNetworkPolicyJob.enabled" . }} +# Enables spec.useMultiNetworkPolicy on network.operator.openshift.io/cluster +# before MultiNetworkPolicy objects sync at wave 37. +# Runs in the default namespace (no NetworkPolicies) because this is a cluster-wide change. +apiVersion: batch/v1 +kind: Job +metadata: + name: enable-multi-network-policy + namespace: {{ .Values.app.udn.multiNetworkPolicyJob.namespace }} + labels: + app: enable-multi-network-policy + annotations: + argocd.argoproj.io/sync-wave: "36" + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + template: + metadata: + name: enable-multi-network-policy + labels: + app: enable-multi-network-policy + spec: + serviceAccountName: enable-multi-network-policy + restartPolicy: OnFailure + containers: + - name: enable-multi-network-policy + image: {{ .Values.app.udn.multiNetworkPolicyJob.image.registry }}/{{ .Values.app.udn.multiNetworkPolicyJob.image.repository }}:{{ .Values.app.udn.multiNetworkPolicyJob.image.tag }} + imagePullPolicy: {{ .Values.app.udn.multiNetworkPolicyJob.image.pullPolicy }} + command: + - /bin/bash + - -c + - | + #!/usr/bin/env bash + set -euo pipefail + + CURRENT="$(oc get network.operator.openshift.io cluster \ + -o jsonpath='{.spec.useMultiNetworkPolicy}' 2>/dev/null || true)" + + if [[ "${CURRENT}" == "true" ]]; then + echo "spec.useMultiNetworkPolicy is already true — no patch needed" + else + echo "Enabling spec.useMultiNetworkPolicy on network.operator.openshift.io/cluster" + oc patch network.operator.openshift.io cluster --type merge \ + -p '{"spec":{"useMultiNetworkPolicy":true}}' + echo "Patch applied successfully" + fi + + {{- if .Values.app.udn.multiNetworkPolicyJob.waitForReconciliation.enabled }} + MAX_RETRIES={{ .Values.app.udn.multiNetworkPolicyJob.waitForReconciliation.maxRetries }} + INTERVAL={{ .Values.app.udn.multiNetworkPolicyJob.waitForReconciliation.intervalSeconds }} + for i in $(seq 1 "${MAX_RETRIES}"); do + if oc api-resources --api-group=k8s.cni.cncf.io 2>/dev/null | grep -q multi-networkpolicies; then + echo "MultiNetworkPolicy API is available (attempt ${i}/${MAX_RETRIES})" + exit 0 + fi + echo "Waiting for CNO to reconcile MultiNetworkPolicy API (${i}/${MAX_RETRIES})..." + sleep "${INTERVAL}" + done + echo "ERROR: timeout waiting for MultiNetworkPolicy API after patch" + exit 1 + {{- else }} + exit 0 + {{- end }} + terminationGracePeriodSeconds: 30 +{{- end }} diff --git a/charts/qtodo/templates/udn-enable-multi-network-policy-rbac.yaml b/charts/qtodo/templates/udn-enable-multi-network-policy-rbac.yaml new file mode 100644 index 00000000..aecf9f18 --- /dev/null +++ b/charts/qtodo/templates/udn-enable-multi-network-policy-rbac.yaml @@ -0,0 +1,43 @@ +{{- if include "qtodo.udn.multiNetworkPolicyJob.enabled" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: enable-multi-network-policy + namespace: {{ .Values.app.udn.multiNetworkPolicyJob.namespace }} + labels: + app: enable-multi-network-policy + annotations: + argocd.argoproj.io/sync-wave: "35" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: enable-multi-network-policy + labels: + app: enable-multi-network-policy + annotations: + argocd.argoproj.io/sync-wave: "35" +rules: + - apiGroups: ["operator.openshift.io"] + resources: ["networks"] + resourceNames: ["cluster"] + verbs: ["get", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: enable-multi-network-policy + labels: + app: enable-multi-network-policy + annotations: + argocd.argoproj.io/sync-wave: "35" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: enable-multi-network-policy +subjects: + - kind: ServiceAccount + name: enable-multi-network-policy + namespace: {{ .Values.app.udn.multiNetworkPolicyJob.namespace }} +{{- end }} diff --git a/charts/qtodo/templates/udn-multi-network-policy.yaml b/charts/qtodo/templates/udn-multi-network-policy.yaml new file mode 100644 index 00000000..c17a2dd6 --- /dev/null +++ b/charts/qtodo/templates/udn-multi-network-policy.yaml @@ -0,0 +1,48 @@ +{{- if and (include "qtodo.udn.enabled" .) .Values.app.udn.networkPolicy.enabled }} +# Default deny on the secondary UDN, then allow egress only to PostgreSQL. +# Router, Vault, OIDC, and DNS stay on the cluster network (qtodo-network-policy). +# Requires spec.useMultiNetworkPolicy: true on network.operator.openshift.io/cluster. +apiVersion: k8s.cni.cncf.io/v1beta1 +kind: MultiNetworkPolicy +metadata: + annotations: + argocd.argoproj.io/sync-wave: "37" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + k8s.v1.cni.cncf.io/policy-for: {{ .Release.Namespace }}/{{ .Values.app.udn.name }} + name: default-deny-udn + namespace: {{ .Release.Namespace }} +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress + ingress: [] + egress: [] +--- +apiVersion: k8s.cni.cncf.io/v1beta1 +kind: MultiNetworkPolicy +metadata: + annotations: + argocd.argoproj.io/sync-wave: "37" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true + k8s.v1.cni.cncf.io/policy-for: {{ .Release.Namespace }}/{{ .Values.app.udn.name }} + name: allow-udn-postgresql + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + app: qtodo + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.postgresql.namespace }} + podSelector: + matchLabels: + app: qtodo-db + ports: + - protocol: TCP + port: {{ .Values.app.udn.networkPolicy.postgresqlPort | default .Values.postgresql.port }} +{{- end }} diff --git a/charts/qtodo/templates/udn-network-attachment-definition.yaml b/charts/qtodo/templates/udn-network-attachment-definition.yaml deleted file mode 100644 index 321a3621..00000000 --- a/charts/qtodo/templates/udn-network-attachment-definition.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if .Values.app.udn.enabled }} -apiVersion: k8s.cni.cncf.io/v1 -kind: NetworkAttachmentDefinition -metadata: - annotations: - argocd.argoproj.io/sync-wave: '36' - description: "Network attachment for qtodo UDN isolation" - name: {{ .Values.app.udn.nadName }} - namespace: {{ .Release.Namespace }} -spec: - config: | - { - "cniVersion": "0.4.0", - "type": "ovn-k8s-cni-overlay", - "name": "{{ .Values.app.udn.name }}", - "topology": "{{ lower .Values.app.udn.topology }}", - "netAttachDefName": "{{ .Release.Namespace }}/{{ .Values.app.udn.nadName }}", - {{- if eq .Values.app.udn.topology "Layer2" }} - "subnets": "{{ .Values.app.udn.subnet }}", - {{- end }} - "mtu": {{ .Values.app.udn.mtu }} - } -{{- end }} diff --git a/charts/qtodo/templates/udn-user-defined-network.yaml b/charts/qtodo/templates/udn-user-defined-network.yaml deleted file mode 100644 index 40ec33d8..00000000 --- a/charts/qtodo/templates/udn-user-defined-network.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.app.udn.enabled }} -apiVersion: k8s.ovn.org/v1 -kind: UserDefinedNetwork -metadata: - annotations: - argocd.argoproj.io/sync-wave: '35' - name: {{ .Values.app.udn.name }} - namespace: {{ .Release.Namespace }} -spec: - topology: {{ .Values.app.udn.topology }} - {{- if eq .Values.app.udn.topology "Layer2" }} - layer2: - role: Primary - subnets: - - {{ .Values.app.udn.subnet }} - {{- if .Values.app.udn.mtu }} - mtu: {{ .Values.app.udn.mtu }} - {{- end }} - {{- else if eq .Values.app.udn.topology "Layer3" }} - layer3: - role: Primary - subnets: - - {{ .Values.app.udn.subnet }} - {{- if .Values.app.udn.joinSubnet }} - joinSubnets: - - {{ .Values.app.udn.joinSubnet }} - {{- end }} - {{- if .Values.app.udn.mtu }} - mtu: {{ .Values.app.udn.mtu }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/qtodo/values.yaml b/charts/qtodo/values.yaml index f820cdbe..2ec989e4 100644 --- a/charts/qtodo/values.yaml +++ b/charts/qtodo/values.yaml @@ -114,58 +114,34 @@ app: # QTodo truststore password path (app-level isolation) vaultPath: "secret/data/apps/qtodo/qtodo-truststore" - # User-Defined Network (UDN) configuration for network isolation - # Provides layer 2/3 network segmentation following Zero Trust principles + # Shared secondary User-Defined Network (CUDN) with qtodo-db. + # The ClusterUserDefinedNetwork CR lives in the qtodo-db chart; this + # chart only attaches the app pod and points JDBC at the static UDN IP. + # Keep name and dbIP in sync with charts/qtodo-db/values.yaml udn.*. udn: enabled: false - # Network name name: qtodo-isolated-network - # NetworkAttachmentDefinition name - nadName: qtodo-udn-nad - # Topology: Layer2 or Layer3 - # Layer2: Same subnet, pods can communicate directly - # Layer3: Different subnets per node, requires routing - topology: Layer2 - # CIDR for the UDN subnet (Layer2 only) - subnet: "10.100.0.0/16" - # Join subnet (for Layer3, optional) - # joinSubnet: "100.64.0.0/16" - # MTU for the network (default: 1400 to avoid fragmentation) - mtu: 1400 - # IPAM configuration - ipam: - type: static - # Network Policy for UDN - # UDN requires policies on both primary (cluster network) and secondary (UDN) interfaces + # Static UDN IP of the PostgreSQL pod (JDBC target) + dbIP: "10.100.0.10" networkPolicy: - # Enable network policies on the UDN + # MultiNetworkPolicy on the UDN (requires CNO useMultiNetworkPolicy: true) enabled: true - # Allowed egress destinations from qtodo pods on the UDN - egress: - # DNS resolution - dns: - enabled: true - port: 5353 - namespace: openshift-dns - # PostgreSQL database - postgresql: - enabled: true - port: 5432 - # Vault for secrets (via primary network, not UDN) - vault: - enabled: true - port: 8200 - namespace: vault - # Allow HTTPS connections to any destination, used for OIDCs (via primary network for external route) - https: - enabled: true - port: 443 - # Allowed ingress sources to qtodo pods on the UDN - ingress: - # OpenShift router (via primary network, not UDN) - router: - enabled: true - port: 8080 + postgresqlPort: 5432 + # Job that patches network.operator.openshift.io/cluster before MultiNetworkPolicy sync. + # Runs in the default namespace (no NetworkPolicies) because this is a cluster-wide change. + # Set enabled to false when another component already manages useMultiNetworkPolicy. + multiNetworkPolicyJob: + enabled: true + namespace: default + image: + registry: registry.redhat.io + repository: openshift4/ose-cli-rhel9 + tag: latest + pullPolicy: IfNotPresent + waitForReconciliation: + enabled: true + maxRetries: 60 + intervalSeconds: 10 # PostgreSQL database configuration postgresql: diff --git a/docs/README.md b/docs/README.md index 06b104f7..3160f0e1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -42,3 +42,4 @@ This directory contains the engineering documentation for the Layered Zero Trust |---|---| | [Private Repositories](private-repos.md) | Deploying the pattern from a private Git repository | | [Confidential Containers](CONFIDENTIAL-CONTAINERS.md) | Confidential Containers integration for trusted execution environments | +| [User-Defined Networks](user-defined-networks.md) | Shared secondary UDN isolating qtodo-db from the cluster network | diff --git a/docs/SYNC-WAVE-INVENTORY.md b/docs/SYNC-WAVE-INVENTORY.md index 4625cab6..3397b899 100644 --- a/docs/SYNC-WAVE-INVENTORY.md +++ b/docs/SYNC-WAVE-INVENTORY.md @@ -48,20 +48,23 @@ Every sync-wave in the repository, in order. **App** = hub-level Argo CD Applica | 34 | └ rhtpa-operator | chart | oidc-cli-secret | | 34 | └ noobaa-mcg | chart | bucket-class | | 35 | rh-keycloak | **App** | | -| 35 | └ qtodo | chart | udn-user-defined-network (UserDefinedNetwork CR for network isolation) | +| 35 | └ qtodo-db | chart | udn-cluster-user-defined-network (ClusterUserDefinedNetwork shared by qtodo and qtodo-db) | | 36 | noobaa-mcg | **App** | | | 36 | └ rhtpa-operator | chart | postgresql-serviceaccount, postgresql-external-secret, object-bucket-claim | | 36 | └ keycloak | chart | keycloak.yaml (Keycloak CR) | | 36 | └ quay-registry | chart | object-bucket-claim | | 36 | └ acs-central | chart | admin-password-secret, central-htpasswd-external-secret, keycloak-client-secret-external-secret | | 36 | └ qtodo-db | chart | postgresql-external-secret | -| 36 | └ qtodo | chart | udn-network-attachment-definition (NAD), truststore-secret-external-secret, registry-external-secret | +| 36 | └ qtodo | chart | truststore-secret-external-secret, registry-external-secret | +| 35 | └ qtodo | chart | enable-multi-network-policy RBAC (SA, ClusterRole, ClusterRoleBinding) | +| 36 | └ qtodo | chart | enable-multi-network-policy (Job in default namespace: patch CNO useMultiNetworkPolicy) | | 38+0 | └ qtodo | chart | registry-seed SA, ClusterRole, ClusterRoleBinding | | 38+5 | └ qtodo | chart (hook) | registry-seed-image (Sync hook Job -- mirrors upstream image to configured registry) | | 37 | qtodo-db | **App** | PostgreSQL for qtodo (before qtodo app) | | 37 | └ quay-registry | chart | quay-s3-setup-serviceaccount (5 resources) | | 37 | └ acs-central | chart | create-htpasswd-field (Job) | -| 37 | └ qtodo | chart | udn-admin-network-policy (AdminNetworkPolicy for UDN traffic control) | +| 37 | └ qtodo-db | chart | udn-multi-network-policy (MultiNetworkPolicy: default-deny UDN + allow PostgreSQL from qtodo) | +| 37 | └ qtodo | chart | udn-multi-network-policy (MultiNetworkPolicy: default-deny UDN + allow egress to PostgreSQL) | | 38 | qtodo | **App** | | | 38 | └ quay-registry | chart | quay-config-bundle-secret | | 39 | └ rhtpa-operator | chart | s3-credentials-secret | @@ -277,6 +280,8 @@ Charts marked **(external)** have been externalized to standalone repositories m | Resource | Old | Current | | --- | ---: | ---: | | postgresql-external-secret.yaml | 5 | 36 | +| udn-cluster-user-defined-network.yaml (ClusterUserDefinedNetwork) | --- | 35 | +| udn-multi-network-policy.yaml (MultiNetworkPolicy) | --- | 37 | | postgresql-statefulset.yaml | 10 | 41 | | postgresql-service.yaml | 10 | 41 | @@ -286,11 +291,11 @@ Charts marked **(external)** have been externalized to standalone repositories m | --- | ---: | ---: | | registry-seed-job.yaml (SA, ClusterRole, ClusterRoleBinding) | --- | 0 | | registry-seed-job.yaml (Sync hook Job) | --- | 5 | -| udn-user-defined-network.yaml | --- | 35 | -| udn-network-attachment-definition.yaml | --- | 36 | | truststore-secret-external-secret.yaml | 5 | 36 | | registry-external-secret.yaml | --- | 36 | -| udn-admin-network-policy.yaml (AdminNetworkPolicy) | --- | 37 | +| udn-enable-multi-network-policy-rbac.yaml (SA, ClusterRole, ClusterRoleBinding) | --- | 35 | +| udn-enable-multi-network-policy-job.yaml (CNO patch Job) | --- | 36 | +| udn-multi-network-policy.yaml (MultiNetworkPolicy) | --- | 37 | | qtodo-truststore-config.yaml | 10 | 41 | | app-deployment.yaml | 20 | 51 | | app-service.yaml | 20 | 51 | diff --git a/docs/USER-JOURNEYS.md b/docs/USER-JOURNEYS.md index 528bab56..5434f534 100644 --- a/docs/USER-JOURNEYS.md +++ b/docs/USER-JOURNEYS.md @@ -81,7 +81,7 @@ Layer 2 contains the applications and workflows that demonstrate Zero Trust prin |---|---|---| | **Data Protection (UC-12)** | qtodo app, SPIFFE/SPIRE, Vault, Keycloak, NetworkPolicies | Multi-tier application (Quarkus + PostgreSQL) where database credentials are fetched just-in-time from Vault using SPIFFE workload identity. User access controlled via Keycloak OIDC. Network segmentation via default-deny policies. | | **Secure Supply Chain (UC-01/UC-02)** | Tekton, RHTAS, RHTPA, ACS, Quay | Automated pipeline that builds, signs (RHTAS/SPIFFE keyless), verifies, generates SBOM, uploads to RHTPA, and deploys with ACS policy checks at every stage. | -| **Network Segmentation** | NetworkPolicies, ACS monitoring | Default-deny NetworkPolicies in qtodo, keycloak-system, vault, and ZTWIM namespaces. Per-pod allow rules with explicit justification. ACS monitors for policy violations. | +| **Network Segmentation** | NetworkPolicies, optional UDN, ACS monitoring | Default-deny NetworkPolicies in qtodo, qtodo-db, keycloak-system, vault, and ZTWIM namespaces. Per-pod allow rules with explicit justification. Optional shared secondary UDN isolates PostgreSQL so qtodo-db is reachable from qtodo only on that network. ACS monitors for policy violations. | | **Runtime Threat Detection** | ACS policies | Custom security policies: suspicious exec detection, runtime privilege escalation prevention, network policy enforcement warnings. | | **Confidential Computing (CoCo)** | hello-coco, Sandboxed Containers, Trustee/KBS | *Status: Configuration exists but NOT functional.* TEE-based workload protection with sealed secrets via KBS. Requires specific hardware (AMD SEV-SNP). Planned for future enablement. | diff --git a/docs/multi-tier.md b/docs/multi-tier.md index a91409f4..2f4ffd54 100644 --- a/docs/multi-tier.md +++ b/docs/multi-tier.md @@ -89,3 +89,9 @@ The qtodo Helm chart provides options to control how and where TLS communication > [!NOTE] > When using `passthrough` termination with custom certificates, ensure the certificate's Subject Alternative Name (SAN) matches the Route hostname. The Service Serving Certificate feature cannot be used in this mode. + +## Optional: User-Defined Network isolation + +By default, qtodo reaches PostgreSQL on the cluster network, limited by `NetworkPolicy` to the `qtodo` namespace. To place the database on an isolated Layer2 segment that only qtodo can join, enable the `udn` feature. qtodo continues to use the cluster network for the router, Vault, and OIDC; JDBC uses the PostgreSQL pod's static address on the shared secondary UDN. + +See [User-Defined Networks](user-defined-networks.md). diff --git a/docs/user-defined-networks.md b/docs/user-defined-networks.md index f6681791..cd0fed8b 100644 --- a/docs/user-defined-networks.md +++ b/docs/user-defined-networks.md @@ -2,102 +2,121 @@ ## Overview -User-Defined Networks (UDN) provide layer 2/3 network isolation for workloads in OpenShift, separate from the default cluster network. This feature implements Zero Trust network segmentation principles by creating an isolated network for the qtodo application, restricting communication to only necessary services. +User-Defined Networks (UDN) provide layer-2 network isolation for workloads in OpenShift, separate from the default cluster network. This feature implements Zero Trust network segmentation for the qtodo multi-tier application now that the frontend (`qtodo`) and PostgreSQL (`qtodo-db`) run in **different namespaces**. + +A cluster-scoped `ClusterUserDefinedNetwork` (CUDN) creates a **shared secondary** Layer2 network that both namespaces join. PostgreSQL is reachable from qtodo **only** on that UDN. qtodo keeps the cluster network as its primary interface for everything else (OpenShift router, Vault, OIDC, DNS). qtodo-db's only permitted cluster-network egress is CoreDNS (`5353/tcp` and `5353/udp`). ## Architecture ### Network Topology -The UDN implementation creates a dedicated isolated network for qtodo workloads: - ```text -┌─────────────────────────────────────────────────────┐ -│ Cluster Network │ -│ ┌────────────┐ ┌─────────┐ ┌──────────┐ │ -│ │ Router │───▶│ qtodo │───▶│ Vault │ │ -│ │ (Ingress) │ │ (eth0) │ │ (8200) │ │ -│ └────────────┘ └────┬────┘ └──────────┘ │ -│ │ │ -│ │ UDN Attachment │ -└─────────────────────────┼───────────────────────────┘ - │ - ┌─────▼──────┐ - │ UDN │ - │ (net1/ │ - │ Layer2) │ - └─────┬──────┘ - │ - ┌─────────┴────────┐ - │ │ - ┌─────▼──────┐ ┌────▼─────┐ - │ qtodo pod │ │ qtodo-db │ - │ (isolated) │──────│ (5432) │ - └────────────┘ └──────────┘ - │ - └─────────▶ DNS (5353) via cluster network +┌──────────────────────────────────────────────────────────────┐ +│ Cluster Network (eth0) │ +│ ┌────────────┐ ┌─────────┐ ┌──────────┐ │ +│ │ Router │────▶│ qtodo │────▶│ Vault │ │ +│ │ (Ingress) │ │ (eth0) │ │ (8200) │ │ +│ └────────────┘ └────┬────┘ └──────────┘ │ +│ │ │ +│ OIDC (443), DNS (5353) │ +│ │ +│ ┌───────────┐ │ +│ │ qtodo-db │──── DNS (5353) only │ +│ │ (eth0) │ PostgreSQL denied │ +│ └───────────┘ │ +└──────────────────────────┬───────────────────────────────────┘ + │ + Shared secondary CUDN (net1, Layer2) + ClusterUserDefinedNetwork + namespaces: qtodo, qtodo-db + │ + ┌─────────────┴─────────────┐ + │ │ + ┌─────▼──────┐ ┌──────▼────────┐ + │ qtodo pod │ │ qtodo-db │ + │ (net1) │─────────────│ (net1, │ + │ │ 5432 │ 10.100.0.10) │ + └────────────┘ └───────────────┘ ``` ### Dual Network Interfaces -When UDN is enabled, qtodo pods have two network interfaces: +| Workload | eth0 (cluster network) | net1 (shared secondary UDN) | +| --- | --- | --- | +| **qtodo** | Ingress from OpenShift router. Egress to Vault (8200), OIDC (443), CoreDNS (5353) | JDBC to PostgreSQL at the static UDN IP (`10.100.0.10:5432`) | +| **qtodo-db** | Egress to CoreDNS (`5353/tcp` and `5353/udp`) only; **no** PostgreSQL ingress | PostgreSQL (`5432/tcp`) from qtodo pods only | -1. **eth0 (Primary - Cluster Network)** - - Ingress from OpenShift Router (port 8080) - - Egress to Vault (SPIFFE auth, port 8200) - - Egress to OIDCs (OIDC back-channel, port 443) - - DNS resolution (CoreDNS, port 5353) +Kubernetes `NetworkPolicy` objects apply only to the cluster network. `MultiNetworkPolicy` objects apply to the CUDN. -2. **net1 (Secondary - UDN)** - - PostgreSQL communication (qtodo ↔ qtodo-db, port 5432) - - Isolated from other cluster workloads - - Layer 2 topology (same subnet across nodes) +Secondary UDNs do **not** support Kubernetes Services. qtodo therefore uses the PostgreSQL pod's static UDN IP as the JDBC host instead of `qtodo-db.qtodo-db.svc`. ## Security Benefits -1. **Network Segmentation**: qtodo workloads are isolated from arbitrary cluster traffic -2. **Explicit Allow-Lists**: AdminNetworkPolicy enforces allow-only-required communication -3. **Defense in Depth**: Combines with existing NetworkPolicy for dual-layer protection -4. **Blast Radius Reduction**: Compromise of qtodo cannot pivot to unrelated services -5. **Compliance**: Supports Zero Trust architecture mandates (NIST 800-207, NIS2, ISO 27001:2022) +1. **Cross-namespace segmentation**: the database namespace is not on a shared cluster-network path to the application +2. **Explicit allow-lists**: cluster-network `NetworkPolicy` plus UDN `MultiNetworkPolicy` +3. **Blast radius reduction**: qtodo-db is completely isolated. It is not accesible from pods other than qtodo pod. +4. **Compliance**: supports Zero Trust architecture mandates (NIST SP 800-207, NIS2, ISO 27001:2022) ## Components -UDN is integrated into the qtodo Helm chart (`charts/qtodo`). When enabled, the following resources are created: +### ClusterUserDefinedNetwork CR -### UserDefinedNetwork CR +Template: `charts/qtodo-db/templates/udn-cluster-user-defined-network.yaml` -Template: `charts/qtodo/templates/udn-user-defined-network.yaml` - -Creates the isolated network with Layer2 topology: +Created in the `qtodo-db` application (sync-wave 35, before the StatefulSet) so both namespaces receive a NAD before pods start: +- Name: `qtodo-isolated-network` +- Topology: Layer2, role **Secondary** (qtodo's primary interface stays the cluster network) - Subnet: `10.100.0.0/16` -- MTU: 1400 (avoids fragmentation) -- IPAM: Persistent IP assignment -- Sync-wave: 35 (before NAD) +- Reserved subnet: `10.100.0.0/28` (static DB IP is taken from this range) +- Namespace selector: `qtodo` and `qtodo-db` (`kubernetes.io/metadata.name`) +- OVN-Kubernetes creates a `NetworkAttachmentDefinition` of the same name in each selected namespace, do not create NADs by hand + +### Pod attachment + +- **qtodo-db** StatefulSet: Multus annotation with a **static** IP (`udn.dbIP`, default `10.100.0.10`) +- **qtodo** Deployment: Multus annotation attaching to the CUDN-managed NAD (IP allocated from the non-reserved range) + +### Cluster-network NetworkPolicy + +When UDN is enabled: -### NetworkAttachmentDefinition +- `qtodo-db-network-policy.yaml` — **no** PostgreSQL ingress on eth0; egress is CoreDNS 5353 only +- `qtodo-network-policy.yaml` — PostgreSQL egress to the `qtodo-db` namespace is **omitted**; router, Vault, OIDC, and DNS stay on eth0 -Template: `charts/qtodo/templates/udn-network-attachment-definition.yaml` +When UDN is disabled, the previous cluster-network PostgreSQL allow rules remain. -Defines how pods attach to the UDN: +### MultiNetworkPolicy (UDN) -- CNI type: `ovn-k8s-cni-overlay` -- References the UserDefinedNetwork -- Used via pod annotation `k8s.v1.cni.cncf.io/networks` -- Sync-wave: 36 (before policies) +Templates: -### AdminNetworkPolicy +- `charts/qtodo-db/templates/udn-multi-network-policy.yaml` +- `charts/qtodo/templates/udn-multi-network-policy.yaml` -Template: `charts/qtodo/templates/udn-admin-network-policy.yaml` +- Default-deny on the UDN in both namespaces +- Allow PostgreSQL (`5432/tcp`) from `app=qtodo` in `qtodo` to `app=qtodo-db` in `qtodo-db` +- qtodo is not allowed to receive traffic on the UDN +- qtodo-db is not allowed UDN egress (DNS stays on eth0) -Explicit allow-list for UDN traffic: +Requires `spec.useMultiNetworkPolicy: true` on `network.operator.openshift.io/cluster`. When `app.udn.networkPolicy.enabled` is `true` (the default) and `app.udn.multiNetworkPolicyJob.enabled` is `true` (the default), the `qtodo` chart runs a Job at sync-wave 36 that patches this setting if needed and waits for the Cluster Network Operator to expose the MultiNetworkPolicy API before wave 37 policies sync. Set `app.udn.multiNetworkPolicyJob.enabled` to `false` when another component already manages `spec.useMultiNetworkPolicy`. If that Cluster Network Operator setting cannot be enabled, set `udn.networkPolicy.enabled` and `app.udn.networkPolicy.enabled` to `false`. UDN membership plus cluster-network `NetworkPolicy` still isolate PostgreSQL. -- **Ingress**: - - OpenShift router (port 8080) - - qtodo pods to qtodo-db (port 5432) -- **Egress**: DNS, PostgreSQL, Vault, Keycloak (HTTPS connections) -- Priority: 50 (higher = processed first) -- Sync-wave: 37 (before qtodo app) +### Cluster Network Operator patch Job + +Template: `charts/qtodo/templates/udn-enable-multi-network-policy-job.yaml` + +The job runs in the `default` namespace, which has no NetworkPolicies, so it can reach the Kubernetes API server without extra egress rules. + +- Runs when `app.udn.enabled`, `app.udn.networkPolicy.enabled`, and `app.udn.multiNetworkPolicyJob.enabled` are all `true` +- Checks `spec.useMultiNetworkPolicy` on `network.operator.openshift.io/cluster` and skips the patch when already `true` +- Waits for the `multi-networkpolicies` API to become available (configurable via `app.udn.multiNetworkPolicyJob.waitForReconciliation`) +- The `qtodo-db` MultiNetworkPolicies (deployed at app-wave 37, before `qtodo` at app-wave 38) carry `SkipDryRunOnMissingResource=true` and will be created successfully once the API is available after the job completes + +Manual patch (troubleshooting only): + +```bash +oc patch network.operator.openshift.io cluster --type merge \ + -p '{"spec":{"useMultiNetworkPolicy":true}}' +``` ## Enabling UDN @@ -108,110 +127,130 @@ python3 scripts/gen-feature-variants.py \ --features udn \ --base values-hub.yaml -# Apply the variant cp /tmp/values-hub-udn.yaml values-hub.yaml ./pattern.sh make install ``` +The fragment sets `udn.enabled` on `qtodo-db` and `app.udn.enabled` on `qtodo`. + ### Option 2: Manual Configuration -1. **Enable UDN in the qtodo application** in `values-hub.yaml`: +Enable **both** applications in `values-hub.yaml`: + +```yaml +clusterGroup: + applications: + qtodo-db: + overrides: + - name: udn.enabled + value: "true" + qtodo: + overrides: + - name: app.udn.enabled + value: "true" +``` + +If another component already manages `spec.useMultiNetworkPolicy` on the cluster network operator, disable the patch Job: - ```yaml - clusterGroup: - applications: - qtodo: - # ... existing config ... - overrides: - # ... existing overrides ... - - name: app.udn.enabled - value: "true" - ``` +```yaml + qtodo: + overrides: + - name: app.udn.multiNetworkPolicyJob.enabled + value: "false" +``` -2. **Deploy**: +Then deploy: - ```bash - ./pattern.sh make install - ``` +```bash +./pattern.sh make install +``` ## Verification -### 1. Check UDN Resources +### 1. Check CUDN and NADs ```bash -# UserDefinedNetwork -oc get userdefinednetwork -n qtodo -NAME AGE -qtodo-isolated-network 5m +oc get clusteruserdefinednetwork qtodo-isolated-network -# NetworkAttachmentDefinition oc get network-attachment-definitions -n qtodo -NAME AGE -qtodo-udn-nad 5m +oc get network-attachment-definitions -n qtodo-db +``` + +Both namespaces should show `qtodo-isolated-network`. + +### 2. Confirm pod attachments + +```bash +oc get pod -n qtodo -l app=qtodo -o jsonpath='{.items[0].metadata.annotations.k8s\.v1\.cni\.cncf\.io/networks}{"\n"}' +oc get pod -n qtodo-db -l app=qtodo-db -o jsonpath='{.items[0].metadata.annotations.k8s\.v1\.cni\.cncf\.io/networks}{"\n"}' + +oc exec -n qtodo deploy/qtodo -c qtodo -- cat /proc/net/fib_trie | grep -B1 '/32 host LOCAL' +oc exec -n qtodo-db qtodo-db-0 -c postgres -- cat /proc/net/fib_trie | grep -B1 '/32 host LOCAL' ``` -### 2. Verify Network Policies +qtodo-db's `net1` address must be `10.100.0.10`. + +### 3. Verify policies ```bash -# AdminNetworkPolicy -oc get adminnetworkpolicy -NAME PRIORITY AGE -qtodo-udn-policy 50 5m +oc get networkpolicy -n qtodo +oc get networkpolicy -n qtodo-db +oc get multi-networkpolicies.k8s.cni.cncf.io -A ``` -### 3. Test Connectivity +### 4. Test connectivity ```bash -# DNS resolution (should work via eth0) -oc exec -n qtodo deploy/qtodo -c qtodo -- getent hosts qtodo-db +# DNS from qtodo via eth0 +oc exec -n qtodo deploy/qtodo -c qtodo -- getent hosts vault.vault.svc + +# PostgreSQL via UDN static IP (should succeed) +oc exec -n qtodo deploy/qtodo -c qtodo -- timeout 5 bash -c '/dev/null' && echo "OK" +# PostgreSQL via cluster-network Service (should fail when UDN is enabled) +oc exec -n qtodo deploy/qtodo -c qtodo -- timeout 5 bash -c ' **Note:** The two features are mutually exclusive — use `entra-id-qtodo` > for qtodo-only setups and `entra-id` when the full supply chain is deployed. +## User-Defined Network isolation (`udn`) + +Isolates PostgreSQL (`qtodo-db`) on a shared Layer2 secondary +`ClusterUserDefinedNetwork`. The qtodo application keeps the cluster +network as its primary interface (router, Vault, OIDC, DNS) and +reaches the database only on the UDN. qtodo-db cluster-network egress +is CoreDNS (`5353/tcp` and `5353/udp`) only. + +```bash +python3 scripts/gen-feature-variants.py --features udn +``` + +This feature has no dependencies. It sets `udn.enabled` on `qtodo-db` +and `app.udn.enabled` on `qtodo`. See +[User-Defined Networks](../docs/user-defined-networks.md) for the +MultiNetworkPolicy prerequisite and verification steps. + ## How It Works 1. The script reads the base `values-hub.yaml`. diff --git a/values-hub.yaml b/values-hub.yaml index 2934d372..d277e204 100644 --- a/values-hub.yaml +++ b/values-hub.yaml @@ -569,6 +569,11 @@ clusterGroup: path: charts/qtodo-db annotations: argocd.argoproj.io/sync-wave: "37" + # Uncomment together with app.udn.enabled on qtodo to isolate + # PostgreSQL on a shared secondary UDN + # overrides: + # - name: udn.enabled + # value: "true" qtodo: name: qtodo namespace: qtodo @@ -598,6 +603,9 @@ clusterGroup: # before the supply-chain pipeline runs (avoids ImagePullBackOff on first install) # - name: app.seedImage.enabled # value: "true" + # Use PostgreSQL on a shared secondary UDN (also set udn.enabled on qtodo-db) + # - name: app.udn.enabled + # value: "true" # Tekton Chains - Configures Tekton Chains for supply chain security # Requires: OpenShift Pipelines operator (openshift-pipelines subscription) # Requires: RHTAS (trusted-artifact-signer) for keyless signing via SPIFFE