Skip to content

[WINC-1971] [ote] Fix proxy test timing issues #4571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 3 commits into from
Sep 8, 2026
+37 −31
Merged

[WINC-1971] [ote] Fix proxy test timing issues #4571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f7995a
[ote] Fix proxy test timing issues
rrasouli Sep 3, 2026
4373399
[ote] Remove dependency on QE-specific Installer-QE-CA certificate in…
rrasouli Sep 7, 2026
408e7d9
[ote] Increase waitWindowsNodesReady timeout for proxy tests
rrasouli Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 22 additions & 25 deletions ote/test/e2e/proxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"

Expand Down Expand Up @@ -65,7 +66,7 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
if restarted {
e2e.Logf("WMCO restarted after trusted CA removal")
winIPs := getWindowsInternalIPs(oc)
waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute)
waitWindowsNodesReady(oc, len(winIPs), 30*time.Minute)
} else {
e2e.Logf("WMCO did not restart after trusted CA removal")
}
Expand All @@ -85,7 +86,7 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
_, err = checkWMCORestarted(oc, timeNoProxy)
o.Expect(err).NotTo(o.HaveOccurred(), "error checking WMCO restart status after noProxy removal")
winIPs := getWindowsInternalIPs(oc)
waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute)
waitWindowsNodesReady(oc, len(winIPs), 30*time.Minute)
noProxyExpected := getEnvVarProxyMap(oc)
waitForProxyOnNodes(oc, winNodes, noProxyExpected)
} else {
Expand All @@ -108,7 +109,7 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
o.Expect(err).NotTo(o.HaveOccurred())

winIPs := getWindowsInternalIPs(oc)
waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute)
waitWindowsNodesReady(oc, len(winIPs), 30*time.Minute)
e2e.Logf("Skipping WICD ConfigMap and node propagation verification - WMCO doesn't remove env vars from ConfigMap")
} else {
e2e.Logf("spec.httpsProxy is not set, skipping removal")
Expand All @@ -122,7 +123,7 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
_, err = checkWMCORestarted(oc, timeNoHttp)
o.Expect(err).NotTo(o.HaveOccurred(), "error checking WMCO restart status after httpProxy removal")
winIPs := getWindowsInternalIPs(oc)
waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute)
waitWindowsNodesReady(oc, len(winIPs), 30*time.Minute)
httpExpected := getEnvVarProxyMap(oc)
waitForProxyOnNodes(oc, winNodes, httpExpected)
} else {
Expand All @@ -144,7 +145,7 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
o.Expect(err).NotTo(o.HaveOccurred(), "error checking WMCO restart status after noProxy update")
winNodes := getWindowsNodeNames(oc)
winIPs := getWindowsInternalIPs(oc)
waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute)
waitWindowsNodesReady(oc, len(winIPs), 30*time.Minute)

g.By("Verify newly added noProxy record exists on WICD Windows Services")
windowsServicesCM, err := popItemFromList(oc, "cm", wicdConfigMap, wmcoNamespace)
Expand Down Expand Up @@ -186,28 +187,24 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
g.SpecTimeout(45*time.Minute),
func(ctx g.SpecContext) {
const (
name = "OCP-68320-custom"
validity = "3650"
caSubj = "/OU=openshift/CN=test-custom-self-cert-signer"
userSelfSignedCommonName = "CN=test-custom-self-cert-signer, OU=openshift"
userInstalledCertCommonName = "CN=Installer-QE-CA, OU=Installer-QE, O=OCP, S=Beijing, C=CN"
namespace = "openshift-config"
configmap = "user-ca-bundle"
name = "OCP-68320-custom"
validity = "3650"
caSubj = "/OU=openshift/CN=test-custom-self-cert-signer"
userSelfSignedCommonName = "CN=test-custom-self-cert-signer, OU=openshift"
namespace = "openshift-config"
configmap = "user-ca-bundle"
)

g.By("Verify that user certificate installed on each Windows worker")
checkUserCertificatesOnNodes(oc, userInstalledCertCommonName, 1)

g.By("Create a self-signed certificate and append to user-ca-bundle")
keyPath := fmt.Sprintf("%s-ca.key", name)
crtPath := fmt.Sprintf("%s-ca.crt", name)
keyPath := filepath.Join(os.TempDir(), fmt.Sprintf("%s-ca.key", name))
crtPath := filepath.Join(os.TempDir(), fmt.Sprintf("%s-ca.crt", name))
defer os.Remove(keyPath)
cmd := fmt.Sprintf("openssl genrsa -out %s-ca.key 4096", name)
cmd := fmt.Sprintf("openssl genrsa -out %s 4096", keyPath)
output, err := exec.Command("bash", "-c", cmd).CombinedOutput()
o.Expect(err).NotTo(o.HaveOccurred(), "failed to generate key: %s", output)

defer os.Remove(crtPath)
cmd = fmt.Sprintf("openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days %s -out %s-ca.crt -subj %s", name, validity, name, caSubj)
cmd = fmt.Sprintf("openssl req -x509 -new -nodes -key %s -sha256 -days %s -out %s -subj %s", keyPath, validity, crtPath, caSubj)
output, err = exec.Command("bash", "-c", cmd).CombinedOutput()
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create certificate: %s", output)

Expand All @@ -222,11 +219,11 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
combinedContent := fmt.Sprintf("%s\n%s", initialConfigMapContent, string(newCertificateContent))
configureCertificateToJSONPatch(oc, combinedContent, configmap, namespace)

g.By("Verify that user certificate installed on each Windows worker")
checkUserCertificatesOnNodes(oc, userInstalledCertCommonName, 1)
g.By("Verify that self-signed certificate synced to each Windows worker")
checkUserCertificatesOnNodes(oc, userSelfSignedCommonName, 1)

g.By("Creating certificate rotation")
cmd = fmt.Sprintf("openssl req -x509 -new -nodes -key %s-ca.key -sha256 -days 1 -out %s-ca.crt -subj %s", name, name, caSubj)
cmd = fmt.Sprintf("openssl req -x509 -new -nodes -key %s -sha256 -days 1 -out %s -subj %s", keyPath, crtPath, caSubj)
output, err = exec.Command("bash", "-c", cmd).CombinedOutput()
o.Expect(err).NotTo(o.HaveOccurred(), "failed to create rotated certificate: %s", output)

Expand All @@ -235,8 +232,8 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
combinedContent = fmt.Sprintf("%s\n%s", initialConfigMapContent, string(newCertificateContent))
configureCertificateToJSONPatch(oc, combinedContent, configmap, namespace)

g.By("Verify that after certificate rotation certificates installed on each Windows worker")
checkUserCertificatesOnNodes(oc, userInstalledCertCommonName, 1)
g.By("Verify that rotated certificate synced to each Windows worker")
checkUserCertificatesOnNodes(oc, userSelfSignedCommonName, 1)

g.By("Verify that self-signed certificate has been removed from each Windows node")
configureCertificateToJSONPatch(oc, initialConfigMapContent, configmap, namespace)
Expand Down Expand Up @@ -280,7 +277,7 @@ var _ = g.Describe("[OTP][sig-windows][apigroup:config.openshift.io] Windows_Con
e2e.Logf("WMCO did not restart after proxy patch, waiting for WICD propagation")
}
winIPs := getWindowsInternalIPs(oc)
waitWindowsNodesReady(oc, len(winIPs), 15*time.Minute)
waitWindowsNodesReady(oc, len(winIPs), 20*time.Minute)

g.By("Verify NO_PROXY changes propagated to Windows nodes")
expectedProxies := getEnvVarProxyMap(oc)
Expand Down
21 changes: 15 additions & 6 deletions ote/test/e2e/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1598,7 +1598,11 @@ func checkWMCORestarted(oc *exutil.CLI, startTime string) (bool, error) {
return false, nil
})
if pollErr != nil {
return false, fmt.Errorf("error restarting WMCO: %v", pollErr)
if pollErr == wait.ErrWaitTimeout {
e2e.Logf("WMCO did not restart within 6 minutes (this is expected for some proxy changes)")
return false, nil
}
return false, fmt.Errorf("error checking WMCO restart: %w", pollErr)
}
return restartDetected, nil
}
Expand Down Expand Up @@ -1707,7 +1711,7 @@ func checkUserCertificatesOnNodes(oc *exutil.CLI, commonName string, expectedCou
e2e.Logf("Waiting for %d user certificate(s) with CN '%s' on node %s", expectedCount, commonName, nodeName)
cmd := fmt.Sprintf("(Get-ChildItem -Path Cert:\\LocalMachine\\Root | Where-Object {$_.Subject -eq '%s'}).Count", commonName)

pollErr := wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) {
pollErr := wait.Poll(10*time.Second, 10*time.Minute, func() (bool, error) {
msg, err := runHostProcessPS(oc, nodeName, windowsDebugImage, cmd)
if err != nil {
e2e.Logf("Error checking certificates on node %s: %v", nodeName, err)
Expand All @@ -1727,7 +1731,7 @@ func checkUserCertificatesOnNodes(oc *exutil.CLI, commonName string, expectedCou
e2e.Logf("Waiting for certificates on node %s: expected %d, found %d", nodeName, expectedCount, numOfCerts)
return false, nil
})
o.Expect(pollErr).NotTo(o.HaveOccurred(), "certificate count did not reach %d on node %s within 5 minutes", expectedCount, nodeName)
o.Expect(pollErr).NotTo(o.HaveOccurred(), "certificate count did not reach %d on node %s within 10 minutes", expectedCount, nodeName)
}
}

Expand All @@ -1748,11 +1752,16 @@ func removeOuterQuotes(s string) string {
}

func configureCertificateToJSONPatch(oc *exutil.CLI, payload, configmap, namespace string) {
// Collapse the blank line introduced when appending a certificate to the existing bundle
payload = strings.Replace(payload, "\n\n", "\n", 1)
jsonPayload := fmt.Sprintf(`{"data":{"ca-bundle.crt":"%s"}}`, strings.ReplaceAll(payload, "\n", ""))
// Marshal the payload so the PEM line breaks are escaped as \n and survive the patch.
// Stripping them instead yields a single-line blob that is not valid PEM, which makes
// WICD reject the whole ca-bundle.crt and import no certificates at all.
var configMapPayload ConfigMapPayload
err := json.Unmarshal([]byte(jsonPayload), &configMapPayload)
o.Expect(err).NotTo(o.HaveOccurred(), "error unmarshalling JSON")
configMapPayload.Data.CaBundleCrt = payload
jsonBytes, err := json.Marshal(configMapPayload)
o.Expect(err).NotTo(o.HaveOccurred(), "error marshalling ConfigMap patch")
jsonPayload := string(jsonBytes)
cmd := oc.AsAdmin().WithoutNamespace().Run("patch").Args("configmap", configmap, "-n", namespace, "-p", jsonPayload)
output, err := cmd.Output()
if err != nil {
Expand Down