From 7faf433e9c683463d633b8463072852e3003ec3c Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Fri, 14 Aug 2026 13:40:21 +1200 Subject: [PATCH 01/11] e2e: tolerate cloud-provider-uninitialized taint on Windows validation pods servercore/nanoserver validation pods (podWindows) had no tolerations, so if the node was Ready but the Azure CCM hadn't yet removed the node.cloudprovider.kubernetes.io/uninitialized:NoSchedule taint, the pod stayed Pending until the 360s wait timed out, failing the test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- e2e/kube.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/e2e/kube.go b/e2e/kube.go index 5a790b74ffc..5b40e56f6c3 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -787,6 +787,17 @@ func podWindows(s *Scenario, podName string, imageName string) *corev1.Pod { Command: []string{"cmd", "/c", "ping", "-t", "localhost"}, }, }, + // Tolerate the cloud-provider-uninitialized taint: the node may report Ready before + // the Azure cloud-controller-manager removes this taint, which would otherwise leave + // this pod Pending (and the test waiting) until the taint clears on its own. + Tolerations: []corev1.Toleration{ + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Operator: corev1.TolerationOpEqual, + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + }, NodeSelector: map[string]string{ "kubernetes.io/hostname": s.Runtime.VM.KubeName, }, From e95adaeb788e9551592ba3d2becbe12ee39fc14e Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Fri, 14 Aug 2026 13:47:04 +1200 Subject: [PATCH 02/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e/kube.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/kube.go b/e2e/kube.go index 5b40e56f6c3..b2801e1ef38 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -793,8 +793,7 @@ func podWindows(s *Scenario, podName string, imageName string) *corev1.Pod { Tolerations: []corev1.Toleration{ { Key: "node.cloudprovider.kubernetes.io/uninitialized", - Operator: corev1.TolerationOpEqual, - Value: "true", + Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, }, }, From 6d917fc5aab1e7f68b67b52be66539d33079870e Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 17 Aug 2026 09:08:45 +1200 Subject: [PATCH 03/11] refactor so windows pods use a daemonset for deployment --- e2e/kube.go | 130 +++++++++++++++++++++++++++++----------------------- 1 file changed, 73 insertions(+), 57 deletions(-) diff --git a/e2e/kube.go b/e2e/kube.go index b2801e1ef38..f26d3075633 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -418,19 +418,10 @@ func daemonsetDebug(ctx context.Context, deploymentName string, nodeSelector map }, }, Spec: corev1.PodSpec{ - HostNetwork: isHostNetwork, - NodeSelector: nodeSelector, - ImagePullSecrets: func() []corev1.LocalObjectReference { - if secretName == "" { - return nil - } - return []corev1.LocalObjectReference{ - { - Name: secretName, - }, - } - }(), - HostPID: true, + HostNetwork: isHostNetwork, + NodeSelector: nodeSelector, + ImagePullSecrets: getImagePullSecrets(secretName), + HostPID: true, Containers: []corev1.Container{ { Image: image, @@ -441,34 +432,49 @@ func daemonsetDebug(ctx context.Context, deploymentName string, nodeSelector map }, }, }, - // Set Tolerations to tolerate the node with test taints "testkey1=value1:NoSchedule,testkey2=value2:NoSchedule". - // This is to ensure that the pod can be scheduled on the node with the taints. - // It won't affect other pods running on the same node. - Tolerations: []corev1.Toleration{ - { - Key: "testkey1", - Operator: corev1.TolerationOpEqual, - Value: "value1", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: "testkey2", - Operator: corev1.TolerationOpEqual, - Value: "value2", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }, - }, + Tolerations: getPodTolerations(), }, }, }, } } +func getImagePullSecrets(secretName string) []corev1.LocalObjectReference { + if secretName == "" { + return nil + } + return []corev1.LocalObjectReference{ + { + Name: secretName, + }, + } +} + +func getPodTolerations() []corev1.Toleration { + // Set Tolerations to tolerate the node with test taints "testkey1=value1:NoSchedule,testkey2=value2:NoSchedule". + // This is to ensure that the pod can be scheduled on the node with the taints. + // It won't affect other pods running on the same node. + return []corev1.Toleration{ + { + Key: "testkey1", + Operator: corev1.TolerationOpEqual, + Value: "value1", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "testkey2", + Operator: corev1.TolerationOpEqual, + Value: "value2", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + } +} + func (k *Kubeclient) ensureProxyConfigMap(ctx context.Context) error { // Minimal HTTP forward proxy in Python. Handles both: // - CONNECT tunneling for HTTPS (curl uses this when HTTPS_PROXY is set) @@ -771,34 +777,44 @@ func podHTTPServerLinux(s *Scenario) *corev1.Pod { } } -func podWindows(s *Scenario, podName string, imageName string) *corev1.Pod { - return &corev1.Pod{ +func podWindows(s *Scenario, podName string, imageName string) *appsv1.DaemonSet { + deploymentName := fmt.Sprintf("%s-test-%s-pod", s.Runtime.VM.KubeName, podName) + return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-test-%s-pod", s.Runtime.VM.KubeName, podName), + Name: deploymentName, Namespace: "default", }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: podName, - Image: imageName, - ImagePullPolicy: "IfNotPresent", - // this should exist on both servercore and nanoserve - Command: []string{"cmd", "/c", "ping", "-t", "localhost"}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": deploymentName, }, }, - // Tolerate the cloud-provider-uninitialized taint: the node may report Ready before - // the Azure cloud-controller-manager removes this taint, which would otherwise leave - // this pod Pending (and the test waiting) until the taint clears on its own. - Tolerations: []corev1.Toleration{ - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": deploymentName, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: podName, + Image: imageName, + ImagePullPolicy: "IfNotPresent", + // this should exist on both servercore and nanoserve + Command: []string{"cmd", "/c", "ping", "-t", "localhost"}, + }, + }, + Tolerations: getPodTolerations(), + NodeSelector: map[string]string{ + "kubernetes.io/hostname": s.Runtime.VM.KubeName, + }, }, - }, - NodeSelector: map[string]string{ - "kubernetes.io/hostname": s.Runtime.VM.KubeName, }, }, } From e3d22bac1042a8ebe34fc0e8d09f1d505b2d515a Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 17 Aug 2026 09:12:20 +1200 Subject: [PATCH 04/11] rename --- e2e/kube.go | 2 +- e2e/test_helpers.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/kube.go b/e2e/kube.go index f26d3075633..445ac9abe01 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -777,7 +777,7 @@ func podHTTPServerLinux(s *Scenario) *corev1.Pod { } } -func podWindows(s *Scenario, podName string, imageName string) *appsv1.DaemonSet { +func debugDaemonsetWindows(s *Scenario, podName string, imageName string) *appsv1.DaemonSet { deploymentName := fmt.Sprintf("%s-test-%s-pod", s.Runtime.VM.KubeName, podName) return &appsv1.DaemonSet{ TypeMeta: metav1.TypeMeta{ diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index edf75233bf4..4a34e7ff231 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -416,12 +416,12 @@ func ValidateNodeCanRunAPod(ctx context.Context, s *Scenario) { if s.IsWindows() { serverCorePods := components.GetServercoreImagesForVHD(s.VHD) for i, pod := range serverCorePods { - ValidatePodRunning(ctx, s, podWindows(s, fmt.Sprintf("servercore%d", i), pod)) + ValidatePodRunning(ctx, s, debugDaemonsetWindows(s, fmt.Sprintf("servercore%d", i), pod)) } nanoServerPods := components.GetNanoserverImagesForVhd(s.VHD) for i, pod := range nanoServerPods { - ValidatePodRunning(ctx, s, podWindows(s, fmt.Sprintf("nanoserver%d", i), pod)) + ValidatePodRunning(ctx, s, debugDaemonsetWindows(s, fmt.Sprintf("nanoserver%d", i), pod)) } } else { ValidatePodRunningWithRetry(ctx, s, podHTTPServerLinux(s), 3) From 06d23aceba9063466ecf64cac94930ed0c14ba16 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 17 Aug 2026 10:23:27 +1200 Subject: [PATCH 05/11] more reuse --- e2e/kube.go | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/e2e/kube.go b/e2e/kube.go index 445ac9abe01..f7ed970027d 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -756,27 +756,18 @@ func podHTTPServerLinux(s *Scenario) *corev1.Pod { // Set Tolerations to tolerate the node with test taints "testkey1=value1:NoSchedule,testkey2=value2:NoSchedule". // This is to ensure that the pod can be scheduled on the node with the taints. // It won't affect other pods running on the same node. - Tolerations: []corev1.Toleration{ - { - Key: "testkey1", - Operator: corev1.TolerationOpEqual, - Value: "value1", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: "testkey2", - Operator: corev1.TolerationOpEqual, - Value: "value2", - Effect: corev1.TaintEffectNoSchedule, - }, - }, - NodeSelector: map[string]string{ - "kubernetes.io/hostname": s.Runtime.VM.KubeName, - }, + Tolerations: getPodTolerations(), + NodeSelector: getNodeSelectorForScenario(s), }, } } +func getNodeSelectorForScenario(s *Scenario) map[string]string { + return map[string]string{ + "kubernetes.io/hostname": s.Runtime.VM.KubeName, + } +} + func debugDaemonsetWindows(s *Scenario, podName string, imageName string) *appsv1.DaemonSet { deploymentName := fmt.Sprintf("%s-test-%s-pod", s.Runtime.VM.KubeName, podName) return &appsv1.DaemonSet{ @@ -810,10 +801,8 @@ func debugDaemonsetWindows(s *Scenario, podName string, imageName string) *appsv Command: []string{"cmd", "/c", "ping", "-t", "localhost"}, }, }, - Tolerations: getPodTolerations(), - NodeSelector: map[string]string{ - "kubernetes.io/hostname": s.Runtime.VM.KubeName, - }, + Tolerations: getPodTolerations(), + NodeSelector: getNodeSelectorForScenario(s), }, }, }, From 06671c346d106c73996ac60425d4d6149745e4cd Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 17 Aug 2026 10:26:17 +1200 Subject: [PATCH 06/11] change to pod spec --- e2e/kube.go | 38 ++++++++++++-------------------------- e2e/test_helpers.go | 4 ++-- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/e2e/kube.go b/e2e/kube.go index f7ed970027d..5e255115047 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -768,9 +768,9 @@ func getNodeSelectorForScenario(s *Scenario) map[string]string { } } -func debugDaemonsetWindows(s *Scenario, podName string, imageName string) *appsv1.DaemonSet { +func debugPodWindows(s *Scenario, podName string, imageName string) *corev1.Pod { deploymentName := fmt.Sprintf("%s-test-%s-pod", s.Runtime.VM.KubeName, podName) - return &appsv1.DaemonSet{ + return &corev1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "DaemonSet", APIVersion: "apps/v1", @@ -779,32 +779,18 @@ func debugDaemonsetWindows(s *Scenario, podName string, imageName string) *appsv Name: deploymentName, Namespace: "default", }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": deploymentName, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": deploymentName, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: podName, - Image: imageName, - ImagePullPolicy: "IfNotPresent", - // this should exist on both servercore and nanoserve - Command: []string{"cmd", "/c", "ping", "-t", "localhost"}, - }, - }, - Tolerations: getPodTolerations(), - NodeSelector: getNodeSelectorForScenario(s), + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: podName, + Image: imageName, + ImagePullPolicy: "IfNotPresent", + // this should exist on both servercore and nanoserve + Command: []string{"cmd", "/c", "ping", "-t", "localhost"}, }, }, + Tolerations: getPodTolerations(), + NodeSelector: getNodeSelectorForScenario(s), }, } } diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index 4a34e7ff231..cbf2c86f9d0 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -416,12 +416,12 @@ func ValidateNodeCanRunAPod(ctx context.Context, s *Scenario) { if s.IsWindows() { serverCorePods := components.GetServercoreImagesForVHD(s.VHD) for i, pod := range serverCorePods { - ValidatePodRunning(ctx, s, debugDaemonsetWindows(s, fmt.Sprintf("servercore%d", i), pod)) + ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("servercore%d", i), pod)) } nanoServerPods := components.GetNanoserverImagesForVhd(s.VHD) for i, pod := range nanoServerPods { - ValidatePodRunning(ctx, s, debugDaemonsetWindows(s, fmt.Sprintf("nanoserver%d", i), pod)) + ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("nanoserver%d", i), pod)) } } else { ValidatePodRunningWithRetry(ctx, s, podHTTPServerLinux(s), 3) From 3aa8decf6d739df3e4b2ff68239e97a5d0a5f8f1 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 17 Aug 2026 10:26:51 +1200 Subject: [PATCH 07/11] retry param --- e2e/test_helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index cbf2c86f9d0..edec25f41f8 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -416,12 +416,12 @@ func ValidateNodeCanRunAPod(ctx context.Context, s *Scenario) { if s.IsWindows() { serverCorePods := components.GetServercoreImagesForVHD(s.VHD) for i, pod := range serverCorePods { - ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("servercore%d", i), pod)) + ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("servercore%d", i), pod), 3) } nanoServerPods := components.GetNanoserverImagesForVhd(s.VHD) for i, pod := range nanoServerPods { - ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("nanoserver%d", i), pod)) + ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("nanoserver%d", i), pod), 3) } } else { ValidatePodRunningWithRetry(ctx, s, podHTTPServerLinux(s), 3) From f209ee35caa073d5c61b27e4b1b1483b4188c867 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 17 Aug 2026 10:30:08 +1200 Subject: [PATCH 08/11] backoff --- e2e/test_helpers.go | 7 ++++--- e2e/validation.go | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index edec25f41f8..bb35c0193d8 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -413,18 +413,19 @@ func maybeSkipScenario(ctx context.Context, t testing.TB, s *Scenario) { } func ValidateNodeCanRunAPod(ctx context.Context, s *Scenario) { + numberRetries := 3 if s.IsWindows() { serverCorePods := components.GetServercoreImagesForVHD(s.VHD) for i, pod := range serverCorePods { - ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("servercore%d", i), pod), 3) + ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("servercore%d", i), pod), numberRetries) } nanoServerPods := components.GetNanoserverImagesForVhd(s.VHD) for i, pod := range nanoServerPods { - ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("nanoserver%d", i), pod), 3) + ValidatePodRunningWithRetry(ctx, s, debugPodWindows(s, fmt.Sprintf("nanoserver%d", i), pod), numberRetries) } } else { - ValidatePodRunningWithRetry(ctx, s, podHTTPServerLinux(s), 3) + ValidatePodRunningWithRetry(ctx, s, podHTTPServerLinux(s), numberRetries) } } diff --git a/e2e/validation.go b/e2e/validation.go index c8f256253ca..bc19de6a920 100644 --- a/e2e/validation.go +++ b/e2e/validation.go @@ -22,8 +22,10 @@ func ValidatePodRunningWithRetry(ctx context.Context, s *Scenario, pod *corev1.P var err error for i := range maxRetries { err = startPodAndCheckItRuns(ctx, s, pod) + // crude expenential backoff: 2s, 4s, 8s, ... + retryBackoff := time.Duration(1< Date: Mon, 17 Aug 2026 10:31:28 +1200 Subject: [PATCH 09/11] log --- e2e/validation.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/validation.go b/e2e/validation.go index bc19de6a920..3a86da499c9 100644 --- a/e2e/validation.go +++ b/e2e/validation.go @@ -23,9 +23,10 @@ func ValidatePodRunningWithRetry(ctx context.Context, s *Scenario, pod *corev1.P for i := range maxRetries { err = startPodAndCheckItRuns(ctx, s, pod) // crude expenential backoff: 2s, 4s, 8s, ... - retryBackoff := time.Duration(1< Date: Mon, 17 Aug 2026 11:36:10 +1200 Subject: [PATCH 10/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e/kube.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/kube.go b/e2e/kube.go index 5e255115047..e321a6822b5 100644 --- a/e2e/kube.go +++ b/e2e/kube.go @@ -772,8 +772,8 @@ func debugPodWindows(s *Scenario, podName string, imageName string) *corev1.Pod deploymentName := fmt.Sprintf("%s-test-%s-pod", s.Runtime.VM.KubeName, podName) return &corev1.Pod{ TypeMeta: metav1.TypeMeta{ - Kind: "DaemonSet", - APIVersion: "apps/v1", + Kind: "Pod", + APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ Name: deploymentName, From ed3d843049682427776bffe273bcb719da839332 Mon Sep 17 00:00:00 2001 From: Tim Wright Date: Mon, 17 Aug 2026 11:42:53 +1200 Subject: [PATCH 11/11] restructyre loop --- e2e/validation.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/e2e/validation.go b/e2e/validation.go index 3a86da499c9..ff387857adf 100644 --- a/e2e/validation.go +++ b/e2e/validation.go @@ -19,19 +19,20 @@ import ( ) func ValidatePodRunningWithRetry(ctx context.Context, s *Scenario, pod *corev1.Pod, maxRetries int) { - var err error - for i := range maxRetries { - err = startPodAndCheckItRuns(ctx, s, pod) + i := 1 + err := startPodAndCheckItRuns(ctx, s, pod) + + for i <= maxRetries && err != nil { // crude expenential backoff: 2s, 4s, 8s, ... retryBackoff := time.Duration(1 << uint(i)) - if err != nil { - s.T.Logf("sleeping %d seconds before retrying pod %q", retryBackoff, pod.Name) - time.Sleep(retryBackoff * time.Second) - s.T.Logf("retrying pod %q validation (%d/%d)", pod.Name, i+1, maxRetries) - continue - } - break + s.T.Logf("sleeping %d seconds before retrying pod %q", retryBackoff, pod.Name) + time.Sleep(retryBackoff * time.Second) + s.T.Logf("retrying pod %q validation (%d/%d)", pod.Name, i+1, maxRetries) + + i++ + err = startPodAndCheckItRuns(ctx, s, pod) } + require.NoErrorf(s.T, err, "failed to validate pod running %q", pod.Name) }