From fec9e6a03eefbc4469039143259bab7279769a36 Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Wed, 12 Aug 2026 16:51:34 -0700 Subject: [PATCH 1/6] feat: add CustomData size guard for hotfix PRs Add a Go-test guard (pkg/agent/customdata_size_guard_test.go) for the scriptless NBC CSE path and wire it into the hotfix-generate workflow. Guard 1 (TestCustomDataSizeWithHotfix) asserts that the slim Mode B CustomData (platform content P + hotfix-injected scripts H) stays under MaxCustomDataLength (87,380). It runs in CI AFTER hotfix_generate.py injects the real H into nodecustomdata.yml, so it measures the actual post-injection payload. Customer certs (R) are moved to protectedSettings and bounded by RP-side per-field cert byte-size validation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 240e2517-2ad6-4925-90fa-4377e01798e6 --- .github/workflows/hotfix-generate.yml | 12 ++++ pkg/agent/customdata_size_guard_test.go | 73 +++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 pkg/agent/customdata_size_guard_test.go diff --git a/.github/workflows/hotfix-generate.yml b/.github/workflows/hotfix-generate.yml index 40174f95cef..217e22ecd15 100644 --- a/.github/workflows/hotfix-generate.yml +++ b/.github/workflows/hotfix-generate.yml @@ -78,6 +78,18 @@ jobs: run: | python3 hotfix/hotfix_generate.py "origin/${{ github.base_ref }}" + # Runs AFTER hotfix injection (so the //go:embed'd nodecustomdata.yml carries the injected + # scripts H) and BEFORE the commit step (so an oversized hotfix fails the PR before its + # content is committed). + # Guard 1 (TestCustomDataSizeWithHotfix): slim Mode B CustomData (P + H) < 87,380. + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: '1.25' + - name: Validate CustomData size + run: | + go test ./pkg/agent/ -run TestCustomDataSizeWithHotfix -v -count=1 + - name: Commit changes via API env: GH_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/pkg/agent/customdata_size_guard_test.go b/pkg/agent/customdata_size_guard_test.go new file mode 100644 index 00000000000..f9ee4404e39 --- /dev/null +++ b/pkg/agent/customdata_size_guard_test.go @@ -0,0 +1,73 @@ +package agent + +import ( + "testing" + + "github.com/Azure/agentbaker/pkg/agent/datamodel" +) + +// CustomData size guard for the scriptless NBC CSE path. +// +// Background: +// +// In Mode B (slim CustomData), the generated payload carries P (platform content) + H (hotfix- +// injected scripts). R (customer certs) is moved to protectedSettings and is bounded by RP-side +// per-field cert byte-size validation, so only H can overflow CustomData. +// +// This guard asserts that the slim Mode B CustomData stays under MaxCustomDataLength (87,380). +// In the hotfix CI it runs AFTER hotfix_generate.py injects the real H into nodecustomdata.yml, +// so the //go:embed'd template already carries H — no synthetic injection needed. + +// newScriptlessGuardConfig builds a minimal scriptless NBC config. +func newScriptlessGuardConfig() *datamodel.NodeBootstrappingConfiguration { + agentPoolProfile := &datamodel.AgentPoolProfile{ + Name: "nodepool1", + OSType: datamodel.Linux, + Distro: datamodel.AKSUbuntuContainerd2204Gen2, + } + return &datamodel.NodeBootstrappingConfiguration{ + ContainerService: &datamodel.ContainerService{ + Location: "eastus", + Properties: &datamodel.Properties{ + OrchestratorProfile: &datamodel.OrchestratorProfile{ + OrchestratorVersion: "1.29.0", + OrchestratorType: datamodel.Kubernetes, + KubernetesConfig: &datamodel.KubernetesConfig{ + ContainerRuntimeConfig: map[string]string{}, + }, + }, + HostedMasterProfile: &datamodel.HostedMasterProfile{ + FQDN: "test-cluster.hcp.eastus.azmk8s.io", + }, + AgentPoolProfiles: []*datamodel.AgentPoolProfile{agentPoolProfile}, + }, + }, + AgentPoolProfile: agentPoolProfile, + CloudSpecConfig: datamodel.AzurePublicCloudSpecForTest, + K8sComponents: &datamodel.K8sComponents{}, + KubeletConfig: map[string]string{}, + EnableScriptlessNBCCSECmd: true, + } +} + +// TestCustomDataSizeWithHotfix (Guard 1): the slim Mode B CustomData (base platform content P +// plus any hotfix-injected scripts H already embedded in nodecustomdata.yml) must stay under +// MaxCustomDataLength. Forcing ScriptlessCSEProvisionMode makes getScriptlessBoothook return the +// slim CustomData directly, so this measures exactly the Mode B CustomData branch. +func TestCustomDataSizeWithHotfix(t *testing.T) { + templateGenerator := InitializeTemplateGenerator() + config := newScriptlessGuardConfig() + // Force Mode B so getScriptlessBoothook early-returns the slim CustomData (P + H). + config.ScriptlessCSEProvisionMode = true + + payload := templateGenerator.getScriptlessBoothook(config) + + if len(payload) >= MaxCustomDataLength { + t.Fatalf("slim Mode B CustomData is %d bytes, must be < MaxCustomDataLength (%d). "+ + "A hotfix injecting too many/too-large scripts (H) overflowed the CustomData limit; "+ + "there is no further fallback once in Mode B, so the node would fail to provision.", + len(payload), MaxCustomDataLength) + } + t.Logf("slim Mode B CustomData: %d / %d bytes (%d headroom)", + len(payload), MaxCustomDataLength, MaxCustomDataLength-len(payload)) +} From de56fd30b78ecc775df4e2f0d822447d395058c2 Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Thu, 13 Aug 2026 10:01:13 -0700 Subject: [PATCH 2/6] add multiple distro --- pkg/agent/customdata_size_guard_test.go | 46 +++++++++++++++++-------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/pkg/agent/customdata_size_guard_test.go b/pkg/agent/customdata_size_guard_test.go index f9ee4404e39..4af69d8b855 100644 --- a/pkg/agent/customdata_size_guard_test.go +++ b/pkg/agent/customdata_size_guard_test.go @@ -18,12 +18,12 @@ import ( // In the hotfix CI it runs AFTER hotfix_generate.py injects the real H into nodecustomdata.yml, // so the //go:embed'd template already carries H — no synthetic injection needed. -// newScriptlessGuardConfig builds a minimal scriptless NBC config. -func newScriptlessGuardConfig() *datamodel.NodeBootstrappingConfiguration { +// newScriptlessGuardConfig builds a minimal scriptless NBC config for the given distro. +func newScriptlessGuardConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { agentPoolProfile := &datamodel.AgentPoolProfile{ Name: "nodepool1", OSType: datamodel.Linux, - Distro: datamodel.AKSUbuntuContainerd2204Gen2, + Distro: distro, } return &datamodel.NodeBootstrappingConfiguration{ ContainerService: &datamodel.ContainerService{ @@ -54,20 +54,38 @@ func newScriptlessGuardConfig() *datamodel.NodeBootstrappingConfiguration { // plus any hotfix-injected scripts H already embedded in nodecustomdata.yml) must stay under // MaxCustomDataLength. Forcing ScriptlessCSEProvisionMode makes getScriptlessBoothook return the // slim CustomData directly, so this measures exactly the Mode B CustomData branch. +// +// The template has distro-conditional blocks (Ubuntu vs Mariner/AzureLinux vs Flatcar vs OSGuard), +// so we table-drive representative distros to ensure no rendering branch exceeds the limit. func TestCustomDataSizeWithHotfix(t *testing.T) { + distros := []struct { + name string + distro datamodel.Distro + }{ + {"Ubuntu2204", datamodel.AKSUbuntuContainerd2204Gen2}, + {"AzureLinuxV2", datamodel.AKSAzureLinuxV2Gen2}, + {"AzureLinuxV3", datamodel.AKSAzureLinuxV3Gen2}, + {"ACL", datamodel.AKSACLGen2TL}, + } + templateGenerator := InitializeTemplateGenerator() - config := newScriptlessGuardConfig() - // Force Mode B so getScriptlessBoothook early-returns the slim CustomData (P + H). - config.ScriptlessCSEProvisionMode = true - payload := templateGenerator.getScriptlessBoothook(config) + for _, d := range distros { + t.Run(d.name, func(t *testing.T) { + config := newScriptlessGuardConfig(d.distro) + // Force Mode B so getScriptlessBoothook early-returns the slim CustomData (P + H). + config.ScriptlessCSEProvisionMode = true + + payload := templateGenerator.getScriptlessBoothook(config) - if len(payload) >= MaxCustomDataLength { - t.Fatalf("slim Mode B CustomData is %d bytes, must be < MaxCustomDataLength (%d). "+ - "A hotfix injecting too many/too-large scripts (H) overflowed the CustomData limit; "+ - "there is no further fallback once in Mode B, so the node would fail to provision.", - len(payload), MaxCustomDataLength) + if len(payload) >= MaxCustomDataLength { + t.Fatalf("slim Mode B CustomData (%s) is %d bytes, must be < MaxCustomDataLength (%d). "+ + "A hotfix injecting too many/too-large scripts (H) overflowed the CustomData limit; "+ + "there is no further fallback once in Mode B, so the node would fail to provision.", + d.name, len(payload), MaxCustomDataLength) + } + t.Logf("slim Mode B CustomData (%s): %d / %d bytes (%d headroom)", + d.name, len(payload), MaxCustomDataLength, MaxCustomDataLength-len(payload)) + }) } - t.Logf("slim Mode B CustomData: %d / %d bytes (%d headroom)", - len(payload), MaxCustomDataLength, MaxCustomDataLength-len(payload)) } From 0122bfbc67b29e582eb65eb74c83b869ae82626c Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Thu, 13 Aug 2026 11:49:58 -0700 Subject: [PATCH 3/6] add flatcar and osguard --- pkg/agent/customdata_size_guard_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/agent/customdata_size_guard_test.go b/pkg/agent/customdata_size_guard_test.go index 4af69d8b855..a3bf74b5de9 100644 --- a/pkg/agent/customdata_size_guard_test.go +++ b/pkg/agent/customdata_size_guard_test.go @@ -65,6 +65,8 @@ func TestCustomDataSizeWithHotfix(t *testing.T) { {"Ubuntu2204", datamodel.AKSUbuntuContainerd2204Gen2}, {"AzureLinuxV2", datamodel.AKSAzureLinuxV2Gen2}, {"AzureLinuxV3", datamodel.AKSAzureLinuxV3Gen2}, + {"AzureLinuxV3OSGuard", datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL}, + {"Flatcar", datamodel.AKSFlatcarGen2}, {"ACL", datamodel.AKSACLGen2TL}, } From 3874d4185457accd84af19297c38e1c147992e6e Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Thu, 13 Aug 2026 13:48:11 -0700 Subject: [PATCH 4/6] add customecloud --- pkg/agent/customdata_size_guard_test.go | 40 +++++++++++++++---------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/pkg/agent/customdata_size_guard_test.go b/pkg/agent/customdata_size_guard_test.go index a3bf74b5de9..705b8ab34ba 100644 --- a/pkg/agent/customdata_size_guard_test.go +++ b/pkg/agent/customdata_size_guard_test.go @@ -55,26 +55,34 @@ func newScriptlessGuardConfig(distro datamodel.Distro) *datamodel.NodeBootstrapp // MaxCustomDataLength. Forcing ScriptlessCSEProvisionMode makes getScriptlessBoothook return the // slim CustomData directly, so this measures exactly the Mode B CustomData branch. // -// The template has distro-conditional blocks (Ubuntu vs Mariner/AzureLinux vs Flatcar vs OSGuard), -// so we table-drive representative distros to ensure no rendering branch exceeds the limit. +// The template has distro-conditional blocks (Ubuntu vs Mariner/AzureLinux vs Flatcar vs OSGuard) +// and a cloud-conditional block (IsAKSCustomCloud inlines init-aks-cloud.sh), so we table-drive +// representative distros × cloud combinations to ensure no rendering branch exceeds the limit. func TestCustomDataSizeWithHotfix(t *testing.T) { - distros := []struct { - name string - distro datamodel.Distro + cases := []struct { + name string + distro datamodel.Distro + customCloud bool }{ - {"Ubuntu2204", datamodel.AKSUbuntuContainerd2204Gen2}, - {"AzureLinuxV2", datamodel.AKSAzureLinuxV2Gen2}, - {"AzureLinuxV3", datamodel.AKSAzureLinuxV3Gen2}, - {"AzureLinuxV3OSGuard", datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL}, - {"Flatcar", datamodel.AKSFlatcarGen2}, - {"ACL", datamodel.AKSACLGen2TL}, + {"Ubuntu2204", datamodel.AKSUbuntuContainerd2204Gen2, false}, + {"Ubuntu2204_CustomCloud", datamodel.AKSUbuntuContainerd2204Gen2, true}, + {"AzureLinuxV2", datamodel.AKSAzureLinuxV2Gen2, false}, + {"AzureLinuxV3", datamodel.AKSAzureLinuxV3Gen2, false}, + {"AzureLinuxV3OSGuard", datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL, false}, + {"Flatcar", datamodel.AKSFlatcarGen2, false}, + {"ACL", datamodel.AKSACLGen2TL, false}, } templateGenerator := InitializeTemplateGenerator() - for _, d := range distros { - t.Run(d.name, func(t *testing.T) { - config := newScriptlessGuardConfig(d.distro) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + config := newScriptlessGuardConfig(tc.distro) + if tc.customCloud { + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{ + Name: "akscustom", + } + } // Force Mode B so getScriptlessBoothook early-returns the slim CustomData (P + H). config.ScriptlessCSEProvisionMode = true @@ -84,10 +92,10 @@ func TestCustomDataSizeWithHotfix(t *testing.T) { t.Fatalf("slim Mode B CustomData (%s) is %d bytes, must be < MaxCustomDataLength (%d). "+ "A hotfix injecting too many/too-large scripts (H) overflowed the CustomData limit; "+ "there is no further fallback once in Mode B, so the node would fail to provision.", - d.name, len(payload), MaxCustomDataLength) + tc.name, len(payload), MaxCustomDataLength) } t.Logf("slim Mode B CustomData (%s): %d / %d bytes (%d headroom)", - d.name, len(payload), MaxCustomDataLength, MaxCustomDataLength-len(payload)) + tc.name, len(payload), MaxCustomDataLength, MaxCustomDataLength-len(payload)) }) } } From c3c6c87c6ed2e0c6ef0c59cc052e05a833bc421c Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Thu, 13 Aug 2026 13:54:33 -0700 Subject: [PATCH 5/6] add a commit sha of setup go v7 --- .github/workflows/hotfix-generate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hotfix-generate.yml b/.github/workflows/hotfix-generate.yml index 217e22ecd15..14eb9c616d7 100644 --- a/.github/workflows/hotfix-generate.yml +++ b/.github/workflows/hotfix-generate.yml @@ -83,7 +83,7 @@ jobs: # content is committed). # Guard 1 (TestCustomDataSizeWithHotfix): slim Mode B CustomData (P + H) < 87,380. - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version: '1.25' - name: Validate CustomData size From 2b007480e93af5b591f85f852fbd58e67f20873e Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Thu, 13 Aug 2026 14:37:23 -0700 Subject: [PATCH 6/6] add enabled_featured.sh --- pkg/agent/customdata_size_guard_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/agent/customdata_size_guard_test.go b/pkg/agent/customdata_size_guard_test.go index 705b8ab34ba..195cdc07108 100644 --- a/pkg/agent/customdata_size_guard_test.go +++ b/pkg/agent/customdata_size_guard_test.go @@ -47,6 +47,10 @@ func newScriptlessGuardConfig(distro datamodel.Distro) *datamodel.NodeBootstrapp K8sComponents: &datamodel.K8sComponents{}, KubeletConfig: map[string]string{}, EnableScriptlessNBCCSECmd: true, + // The hotfix flow sets ENABLE_PROVISIONING_HOTFIX=true, which adds enabled_features.sh + // to the Mode B payload via renderEnabledFeatures. Include it so the guard measures + // the same payload size as a real hotfix-enabled node. + EnabledFeatures: map[string]string{"ENABLE_PROVISIONING_HOTFIX": "true"}, } }