diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 5ad1fb5f9c1..02895ba294d 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -192,11 +192,6 @@ function basePrep { logs_to_events "AKS.CSE.fetch_and_cache_imds_instance_metadata" fetch_and_cache_imds_instance_metadata - # This function creates the /etc/kubernetes/azure.json file. It also creates the custom - # cloud configuration file if running in a custom cloud environment. - logs_to_events "AKS.CSE.configureAzureJson" configureAzureJson - - logs_to_events "AKS.CSE.ensureKubeCACert" ensureKubeCACert logs_to_events "AKS.CSE.installSecureTLSBootstrapClient" installSecureTLSBootstrapClient @@ -422,6 +417,9 @@ EOF # After this stage the node should be fully integrated into the cluster. # IMPORTANT: This stage should only run when actually joining a node to the cluster. This step should not be run when creating a VHD image function nodePrep { + logs_to_events "AKS.CSE.configureAzureJson" configureAzureJson + logs_to_events "AKS.CSE.ensureKubeCACert" ensureKubeCACert + logs_to_events "AKS.CSE.fetch_and_cache_imds_instance_metadata" fetch_and_cache_imds_instance_metadata reconcileVulnerableKernelModuleMitigation diff --git a/parts/windows/kuberneteswindowssetup.pisphases.tests.ps1 b/parts/windows/kuberneteswindowssetup.pisphases.tests.ps1 new file mode 100644 index 00000000000..281a2501430 --- /dev/null +++ b/parts/windows/kuberneteswindowssetup.pisphases.tests.ps1 @@ -0,0 +1,143 @@ +# A node booted from a PIS-cached VHD skips BasePrep, because base_prep.complete is already in the +# image. Every setting that describes the cluster, the node identity or a credential must therefore +# be written in NodePrep, or the node runs on values captured when the image was baked. +# These checks read the template as text so they pin the phase a call is made from. + +BeforeAll { + $script:TemplatePath = Join-Path $PSScriptRoot 'kuberneteswindowssetup.ps1.template' + $script:TemplateText = Get-Content -Path $script:TemplatePath -Raw + + function Get-FunctionBody { + param( + [Parameter(Mandatory = $true)][string] $Text, + [Parameter(Mandatory = $true)][string] $Name + ) + + $lines = $Text -split "`r?`n" + $start = -1 + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -match "^\s*function\s+$Name\s*\{") { + $start = $i + break + } + } + if ($start -lt 0) { + throw "function $Name not found in template" + } + + $depth = 0 + $body = New-Object System.Collections.Generic.List[string] + for ($i = $start; $i -lt $lines.Count; $i++) { + $line = $lines[$i] + $depth += ([regex]::Matches($line, '\{')).Count + $depth -= ([regex]::Matches($line, '\}')).Count + $body.Add($line) + if ($depth -le 0 -and $i -gt $start) { + break + } + } + return ($body -join "`n") + } + + $script:BasePrepBody = Get-FunctionBody -Text $script:TemplateText -Name 'BasePrep' + $script:NodePrepBody = Get-FunctionBody -Text $script:TemplateText -Name 'NodePrep' +} + +Describe 'Windows CSE PIS phase placement' { + # Get-FunctionBody tracks brace depth across all text, including braces inside strings and + # comments, so a miscount could truncate a body early and make the negative assertions below + # pass for the wrong reason. Anchor on the closing log line of each phase so a truncated + # extraction fails loudly here first. + Context 'extraction integrity' { + It 'extracts the whole BasePrep body' { + $script:BasePrepBody | Should -Match 'function BasePrep' + $script:BasePrepBody | Should -Match 'BasePrep completed successfully' + } + + It 'extracts the whole NodePrep body' { + $script:NodePrepBody | Should -Match 'function NodePrep' + $script:NodePrepBody | Should -Match 'NodePrep completed successfully' + } + + It 'does not bleed one phase into the other' { + $script:BasePrepBody | Should -Not -Match 'function NodePrep' + $script:NodePrepBody | Should -Not -Match 'function BasePrep' + } + } + + Context 'phase gate' { + It 'skips BasePrep when the image already carries the marker' { + $script:TemplateText | Should -Match 'base_prep\.complete' + $script:TemplateText | Should -Match 'if\s*\(-not\s*\(Test-Path\s*"C:\\AzureData\\base_prep\.complete"\)\)' + } + + It 'runs NodePrep on every node that is not a pre-provision bake' { + $script:TemplateText | Should -Match 'if\s*\(-not\s*\$PreProvisionOnly\)' + } + + It 'writes the marker only for a pre-provision bake' { + $script:TemplateText | Should -Match '\$PreProvisionOnly.*base_prep\.complete' + } + } + + Context 'cloud provider config' { + # azure.json carries the service principal secret, the user assigned identity and the + # VMSS, subnet, NSG, VNet and route table this node belongs to. Writing it in BasePrep + # would bake one cluster's identity and network into an image reused by other nodes. + It 'writes azure.json in NodePrep' { + $script:NodePrepBody | Should -Match 'Write-AzureConfig' + } + + It 'does not write azure.json in BasePrep' { + $script:BasePrepBody | Should -Not -Match 'Write-AzureConfig' + } + + It 'writes azure.json before kubelet is installed and started' { + $configIndex = $script:NodePrepBody.IndexOf('Write-AzureConfig') + $kubeletIndex = $script:NodePrepBody.IndexOf('Install-KubernetesServices') + $configIndex | Should -BeGreaterThan -1 + $kubeletIndex | Should -BeGreaterThan -1 + $configIndex | Should -BeLessThan $kubeletIndex + } + } + + Context 'credentials and cluster identity' { + It 'writes the bootstrap kubeconfig in NodePrep only' { + $script:NodePrepBody | Should -Match 'Write-BootstrapKubeConfig' + $script:BasePrepBody | Should -Not -Match 'Write-BootstrapKubeConfig' + } + + It 'writes the client kubeconfig in NodePrep only' { + $script:NodePrepBody | Should -Match 'Write-KubeConfig' + $script:BasePrepBody | Should -Not -Match 'Write-KubeConfig\b' + } + + It 'writes the cluster CA certificate in NodePrep only' { + $script:NodePrepBody | Should -Match 'Write-CACert' + $script:BasePrepBody | Should -Not -Match 'Write-CACert' + } + } + + Context 'cluster network config' { + It 'writes the Azure CNI config in NodePrep only' { + $script:NodePrepBody | Should -Match 'Set-AzureCNIConfig' + $script:BasePrepBody | Should -Not -Match 'Set-AzureCNIConfig' + } + + # BasePrep writes placeholders so the cached CSE scripts can read the pause image while + # configuring containerd during the bake. NodePrep must rewrite the file so the values a + # cached image carries are replaced with this cluster's. + It 'rewrites the kube cluster config in NodePrep after writing placeholders in BasePrep' { + $script:BasePrepBody | Should -Match 'Write-KubeClusterConfig' + $script:NodePrepBody | Should -Match 'Write-KubeClusterConfig' + } + + It 'refreshes the kubelet serving certificate config before the kube cluster config is written' { + $rotationIndex = $script:NodePrepBody.IndexOf('Configure-KubeletServingCertificateRotation') + $clusterConfigIndex = $script:NodePrepBody.IndexOf('Write-KubeClusterConfig') + $rotationIndex | Should -BeGreaterThan -1 + $clusterConfigIndex | Should -BeGreaterThan -1 + $rotationIndex | Should -BeLessThan $clusterConfigIndex + } + } +} diff --git a/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh new file mode 100644 index 00000000000..de17331308b --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# PIS nodes skip basePrep. Per-cluster files must be written from provisioning CustomData in +# nodePrep and must not be captured in the image. + +Describe 'cse_main.sh PIS-safe configuration' + CSE_MAIN="./parts/linux/cloud-init/artifacts/cse_main.sh" + + phase_body() { + awk -v name="${1}" ' + $0 ~ "^function " name "[[:space:]]*[{]" { inside = 1 } + inside { print } + inside && /^}/ { exit } + ' "${CSE_MAIN}" + } + + code_lines() { + grep -v '^[[:space:]]*#' + } + + phase_count() { + phase_body "${1}" | code_lines | grep -c -- "${2}" || true + } + + script_count() { + code_lines < "${CSE_MAIN}" | grep -c -- "${1}" || true + } + + node_prep_event_calls() { + phase_body "nodePrep" | code_lines | + sed -n 's/^[[:space:]]*logs_to_events "[^"]*" \([A-Za-z_][A-Za-z0-9_]*\).*/\1/p' + } + + dispatch_calls() { + awk ' + $0 == "if [ ! -f /opt/azure/containers/base_prep.complete ]; then" { inside = 1 } + inside { print } + ' "${CSE_MAIN}" | + code_lines | + grep -E '^if .*base_prep.complete|^[[:space:]]*(basePrep|nodePrep)$|^if .*PRE_PROVISION_ONLY' + } + + Describe 'cloud provider config and cluster CA' + It 'does not write per-cluster files in basePrep' + base_prep_counts() { + phase_count "basePrep" "configureAzureJson" + phase_count "basePrep" "ensureKubeCACert" + } + When call base_prep_counts + The line 1 of output should equal "0" + The line 2 of output should equal "0" + End + + It 'writes each per-cluster file exactly once in nodePrep' + node_prep_counts() { + phase_count "nodePrep" "configureAzureJson" + phase_count "nodePrep" "ensureKubeCACert" + } + When call node_prep_counts + The line 1 of output should equal "1" + The line 2 of output should equal "1" + End + + It 'has no duplicate writer elsewhere in the script' + writer_totals() { + script_count "configureAzureJson" + script_count "ensureKubeCACert" + } + When call writer_totals + The line 1 of output should equal "1" + The line 2 of output should equal "1" + End + + It 'writes azure.json and ca.crt before other nodePrep event calls' + When call node_prep_event_calls + The line 1 of output should equal "configureAzureJson" + The line 2 of output should equal "ensureKubeCACert" + End + + It 'writes azure.json before secure TLS bootstrap consumes it' + node_prep_order() { + phase_body "nodePrep" | code_lines | grep -n -E 'configureAzureJson|configureAndEnableSecureTLSBootstrapping' + } + When call node_prep_order + The line 1 of output should include "configureAzureJson" + The line 2 of output should include "configureAndEnableSecureTLSBootstrapping" + End + + It 'writes ca.crt before the API server check consumes it' + node_prep_order() { + phase_body "nodePrep" | code_lines | grep -n -E 'ensureKubeCACert|cacert /etc/kubernetes/certs/ca.crt' + } + When call node_prep_order + The line 1 of output should include "ensureKubeCACert" + The line 2 of output should include "cacert /etc/kubernetes/certs/ca.crt" + End + + It 'writes azure.json before kubelet starts' + node_prep_order() { + phase_body "nodePrep" | code_lines | grep -n -E 'configureAzureJson|ensureKubelet$' + } + When call node_prep_order + The line 1 of output should include "configureAzureJson" + The line 2 of output should include "ensureKubelet" + End + End + + Describe 'stage gate' + It 'skips basePrep for cached images and nodePrep for image creation' + When call dispatch_calls + The line 1 of output should equal 'if [ ! -f /opt/azure/containers/base_prep.complete ]; then' + The line 2 of output should equal ' basePrep' + The line 3 of output should equal 'if [ "${PRE_PROVISION_ONLY}" != "true" ]; then' + The line 4 of output should equal ' nodePrep' + The lines of output should equal 4 + End + End +End