Fall back to leader address when joining controllers (#1103)

phase/install_controllers.go

@@ -129,16 +129,40 @@ func (p *InstallControllers) Run(ctx context.Context) error {
 			h.Metadata.K0sTokenData.URL = p.Config.Spec.KubeAPIURL()
 		}
 	}
+	// The freshly initialized leader is the only controller guaranteed to be up
+	// and reachable during this phase, so it is the safe fallback join target if
+	// the token-embedded URL (derived from spec.api.externalAddress) cannot be
+	// reached - e.g. an externally managed keepalived VIP not currently assigned
+	// to the leader.
+	leaderJoinHost := p.leader.PrivateAddress
+	if leaderJoinHost == "" {
+		leaderJoinHost = p.leader.Address()
+	}
+
 	err := p.parallelDo(ctx, p.hosts, func(_ context.Context, h *cluster.Host) error {
-		if p.IsWet() || !p.leader.Metadata.DryRunFakeLeader {
-			log.Infof("%s: validating api connection to %s", h, h.Metadata.K0sTokenData.URL)
-			if err := retry.WithDefaultTimeout(ctx, node.HTTPStatusFunc(h, h.Metadata.K0sTokenData.URL, 200, 401, 404)); err != nil {
+		if !p.IsWet() && p.leader.Metadata.DryRunFakeLeader {
+			log.Warnf("%s: dry-run: skipping api connection validation because cluster is not actually running", h)
+			return nil
+		}
+
+		log.Infof("%s: validating api connection to %s", h, h.Metadata.K0sTokenData.URL)
+		if err := retry.WithDefaultTimeout(ctx, node.HTTPStatusFunc(h, h.Metadata.K0sTokenData.URL, 200, 401, 404)); err == nil {
+			return nil
+		} else {
+			// Fall back to joining via the leader's own address. Rewrite the
+			// token so the actual join targets the leader too, not just this
+			// preflight check.
+			rewritten, rerr := h.Metadata.K0sTokenData.WithJoinHost(leaderJoinHost)
+			if rerr != nil {
 				return fmt.Errorf("failed to connect from controller to kubernetes api - check networking: %w", err)
 			}
-		} else {
-			log.Warnf("%s: dry-run: skipping api connection validation to because cluster is not actually running", h)
+			log.Warnf("%s: api connection to %s failed, retrying via leader address %s", h, h.Metadata.K0sTokenData.URL, rewritten.URL)
+			if rerr := retry.WithDefaultTimeout(ctx, node.HTTPStatusFunc(h, rewritten.URL, 200, 401, 404)); rerr != nil {
+				return fmt.Errorf("failed to connect from controller to kubernetes api - check networking: %w", err)
+			}
+			h.Metadata.K0sTokenData = rewritten
+			return nil
 		}
-		return nil
 	})
 	if err != nil {
 		return err

pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go

@@ -1,11 +1,14 @@
 package cluster
 
 import (
+	"bytes"
 	"compress/gzip"
 	"context"
 	"encoding/base64"
 	"fmt"
 	"io"
+	"net"
+	"net/url"
 	"strings"
 	"time"
 
@@ -181,6 +184,88 @@ type TokenData struct {
 	Kubeconfig []byte
 }
 
+// WithJoinHost returns a copy of the token data with the embedded kubeconfig's
+// cluster server host replaced by host (scheme and port are preserved) and the
+// token re-encoded. Because an actual controller/worker join reads the server
+// URL from the token file, rewriting the host here redirects the join itself,
+// not just k0sctl's preflight validation.
+//
+// This is used to make joins target a known-reachable address (the freshly
+// initialized leader) when the token-embedded address derived from
+// spec.api.externalAddress is not reachable during bootstrap, e.g. an
+// externally managed keepalived VIP that is not currently assigned to the
+// leader.
+//
+// Note: the leader's serving certificate must include host in its SANs or the
+// join's TLS verification will fail.
+func (d TokenData) WithJoinHost(host string) (TokenData, error) {
+	if len(d.Kubeconfig) == 0 {
+		return d, fmt.Errorf("token has no kubeconfig data to rewrite")
+	}
+
+	cfg := dig.Mapping{}
+	if err := yaml.Unmarshal(d.Kubeconfig, &cfg); err != nil {
+		return d, fmt.Errorf("failed to unmarshal token: %w", err)
+	}
+
+	clusters, ok := cfg.Dig("clusters").([]any)
+	if !ok || len(clusters) < 1 {
+		return d, fmt.Errorf("failed to find clusters in token")
+	}
+	cluster, ok := clusters[0].(dig.Mapping)
+	if !ok {
+		return d, fmt.Errorf("failed to find cluster in token")
+	}
+	clusterData, ok := cluster.Dig("cluster").(dig.Mapping)
+	if !ok {
+		return d, fmt.Errorf("failed to find cluster data in token")
+	}
+	server, ok := clusterData["server"].(string)
+	if !ok || server == "" {
+		return d, fmt.Errorf("failed to find cluster url in token")
+	}
+
+	u, err := url.Parse(server)
+	if err != nil {
+		return d, fmt.Errorf("failed to parse cluster url %q: %w", server, err)
+	}
+	if port := u.Port(); port != "" {
+		u.Host = net.JoinHostPort(host, port)
+	} else {
+		u.Host = host
+	}
+	clusterData["server"] = u.String()
+
+	kubeconfig, err := yaml.Marshal(cfg)
+	if err != nil {
+		return d, fmt.Errorf("failed to marshal token: %w", err)
+	}
+	token, err := encodeToken(kubeconfig)
+	if err != nil {
+		return d, err
+	}
+
+	return TokenData{ID: d.ID, URL: u.String(), Token: token, Kubeconfig: kubeconfig}, nil
+}
+
+// encodeToken encodes a kubeconfig into a k0s join token (gzip + base64), the
+// inverse of the decoding performed by ParseToken.
+func encodeToken(kubeconfig []byte) (string, error) {
+	var buf bytes.Buffer
+	b64 := base64.NewEncoder(base64.StdEncoding, &buf)
+	gz := gzip.NewWriter(b64)
+	if _, err := gz.Write(kubeconfig); err != nil {
+		return "", fmt.Errorf("failed to compress token: %w", err)
+	}
+	if err := gz.Close(); err != nil {
+		return "", fmt.Errorf("failed to compress token: %w", err)
+	}
+	if err := b64.Close(); err != nil {
+		return "", fmt.Errorf("failed to encode token: %w", err)
+	}
+	return buf.String(), nil
+}
+
 // ParseToken returns TokenData for a token string
 func ParseToken(s string) (TokenData, error) {
 	data := TokenData{Token: s}

pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s_test.go

@@ -1,6 +1,7 @@
 package cluster
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/creasty/defaults"
@@ -10,15 +11,55 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-func TestParseToken(t *testing.T) {
-	token := "H4sIAAAAAAAC/2xVXY/iOBZ9r1/BH6geO4GeAWkfKiEmGGLKjn1N/BbidAFOgjuk+Frtf18V3SPtSvN2fc/ROdaVfc9L6Q9Q9+fDqZuNLvilaj7PQ92fZy+vo9/17GU0Go3OdX+p+9loPwz+PPvjD/xn8A3/+Q19C2bfx+Pwyanqfjj8OFTlUL+Wn8P+1B+G+6sth3I2WudoWOc4FspSeYjmAqjKlaEcESWeGBpih2muRCQSNucavEEkzBWNDGoApDV1t19W6uNSbJsyRzS1mPc7TVdiDknV0qNFQmjl1zvsaZmao3RECHVd8YZEFtlEgGW8ISmXBIQiY6km+wwbr5v9yoIvVHs71pL81CAio0yYpQ2DJMFSe1InWHEZMZHQveiqa/3hf2Eg+v/FpKJdnZifHCA2aKK5IwwSsbVzYnZgJkWLdUZ8IbfCZA5CE1hSKhxliZ2rkKRxw2hxZIlSEHMgwFWCckUTi8iTmyNy+ZqJUtktO2Y9C8Wpuk8DsTUT7ehnjt9uBTQ0T7yDB9nyw+A4Tlb5wt2NbHgB5LSJpwvR2Ytpp6oKm/lG2ZvUZoDERjs9vubzamxJcZEaX6vDwLKWFeUWIoOqi7z/hWx7c2q77DfcJ5BkQQFAyxYw6xix8BZILAar8Ha3GM7l420ssZ/UZE/rrQtUytSus4ssXGKOissKkdgiOskw1fowPKRqxnFLPy0hj1pPvV6IC0t4AOhGgZDlZjFdGYdXLBVZBozKrUccW6Ra2mQNm5sF9bsHXRVqv8lB7E3XmNyZjKHTSm7Jp82HyxoJDom56HY8zgFa6/xCoOtdIL8qF8t71rDUYBZAI247ZHnpiluZn+9WNu8GsvEusFuOpvNS20J/+GUN1aN2U2kfpFQouVaBj3PsW6VgXwXVeJfSd4DlLdN2JR+gqoAed8hEBcB7OXc4J3Dl2jLuSCQCL0pHo9jhiCU2ygCcSC3hh2moFEQWNTFvfaQS2snGLJXDMdfFWCiquBKRUh8XqZZXgZIbaJEYTLbcUQnBtLDkY8VbWuzmMAhH97ka1tWWKN1lvQFLICEb3tq+0vu+VNXEPqKvN/gQjkQSsejLv3BsUjTRNk8mpNbMF46d1Ju/SURPRWihBOJtS5eVwp9ZQhvIB8+UCo1ksSXg7IPcS2wNc35cphHKVKNE4rebbSR2ODpxd5uYAA/VfH+JW9Jt1GRv231eJ9mj1uao2+Z7pRrB2ulP4+xF5kOxDtUF3PLKJXmXCb4XgQmzuRFVmmGZnCaA/nrIBdCvuRduvMpVs8lcNi7UcDVhRG0A93JLYpP66yqYgJoLoZumlQ9x2xFD8znIkux77oacdWqSdZSVyjCWnkKmb+9WDz/Nh5+b9O1SIDIUHaC6bW5V4qFsYSnSRmUIloXCuV1MaE7IsQAxBkR5ndqASRZtFDVGm7VszHGzwEfhJqzUzTV2tMi1iG369dfsmjVvkxKKfhMPgjsccEUPLMmCTcJCsTDrfGHGdXsOJcBpo4ezQd7sQroC3EQrdLtVD+Z16lZCY58rEO8SrX7vZiId/+AIckiaRa5YBIl67uU1P/3rZTTqyraejRw6v1Snbqhvw6+U+FX/Som/I+PJ+mp8np+nz13d1MPr7nQazkNf+v9X++z7uhte/1Z6Nt2hs7NRfOp+HD5efF//qPu6q+rzbPTv/7x8qT7Nf4v8g/zT+HmF4eTqbjY6fD+E949vVzeZ7vHx8mM6uPCATi//DQAA//+MVAsnAgcAAA=="
+// testJoinToken is a sample (expired, non-secret) k0s join token fixture whose
+// embedded kubeconfig points at https://172.17.0.2:6443 with token ID i6i3yg.
+const testJoinToken = "H4sIAAAAAAAC/2xVXY/iOBZ9r1/BH6geO4GeAWkfKiEmGGLKjn1N/BbidAFOgjuk+Frtf18V3SPtSvN2fc/ROdaVfc9L6Q9Q9+fDqZuNLvilaj7PQ92fZy+vo9/17GU0Go3OdX+p+9loPwz+PPvjD/xn8A3/+Q19C2bfx+Pwyanqfjj8OFTlUL+Wn8P+1B+G+6sth3I2WudoWOc4FspSeYjmAqjKlaEcESWeGBpih2muRCQSNucavEEkzBWNDGoApDV1t19W6uNSbJsyRzS1mPc7TVdiDknV0qNFQmjl1zvsaZmao3RECHVd8YZEFtlEgGW8ISmXBIQiY6km+wwbr5v9yoIvVHs71pL81CAio0yYpQ2DJMFSe1InWHEZMZHQveiqa/3hf2Eg+v/FpKJdnZifHCA2aKK5IwwSsbVzYnZgJkWLdUZ8IbfCZA5CE1hSKhxliZ2rkKRxw2hxZIlSEHMgwFWCckUTi8iTmyNy+ZqJUtktO2Y9C8Wpuk8DsTUT7ehnjt9uBTQ0T7yDB9nyw+A4Tlb5wt2NbHgB5LSJpwvR2Ytpp6oKm/lG2ZvUZoDERjs9vubzamxJcZEaX6vDwLKWFeUWIoOqi7z/hWx7c2q77DfcJ5BkQQFAyxYw6xix8BZILAar8Ha3GM7l420ssZ/UZE/rrQtUytSus4ssXGKOissKkdgiOskw1fowPKRqxnFLPy0hj1pPvV6IC0t4AOhGgZDlZjFdGYdXLBVZBozKrUccW6Ra2mQNm5sF9bsHXRVqv8lB7E3XmNyZjKHTSm7Jp82HyxoJDom56HY8zgFa6/xCoOtdIL8qF8t71rDUYBZAI247ZHnpiluZn+9WNu8GsvEusFuOpvNS20J/+GUN1aN2U2kfpFQouVaBj3PsW6VgXwXVeJfSd4DlLdN2JR+gqoAed8hEBcB7OXc4J3Dl2jLuSCQCL0pHo9jhiCU2ygCcSC3hh2moFEQWNTFvfaQS2snGLJXDMdfFWCiquBKRUh8XqZZXgZIbaJEYTLbcUQnBtLDkY8VbWuzmMAhH97ka1tWWKN1lvQFLICEb3tq+0vu+VNXEPqKvN/gQjkQSsejLv3BsUjTRNk8mpNbMF46d1Ju/SURPRWihBOJtS5eVwp9ZQhvIB8+UCo1ksSXg7IPcS2wNc35cphHKVKNE4rebbSR2ODpxd5uYAA/VfH+JW9Jt1GRv231eJ9mj1uao2+Z7pRrB2ulP4+xF5kOxDtUF3PLKJXmXCb4XgQmzuRFVmmGZnCaA/nrIBdCvuRduvMpVs8lcNi7UcDVhRG0A93JLYpP66yqYgJoLoZumlQ9x2xFD8znIkux77oacdWqSdZSVyjCWnkKmb+9WDz/Nh5+b9O1SIDIUHaC6bW5V4qFsYSnSRmUIloXCuV1MaE7IsQAxBkR5ndqASRZtFDVGm7VszHGzwEfhJqzUzTV2tMi1iG369dfsmjVvkxKKfhMPgjsccEUPLMmCTcJCsTDrfGHGdXsOJcBpo4ezQd7sQroC3EQrdLtVD+Z16lZCY58rEO8SrX7vZiId/+AIckiaRa5YBIl67uU1P/3rZTTqyraejRw6v1Snbqhvw6+U+FX/Som/I+PJ+mp8np+nz13d1MPr7nQazkNf+v9X++z7uhte/1Z6Nt2hs7NRfOp+HD5efF//qPu6q+rzbPTv/7x8qT7Nf4v8g/zT+HmF4eTqbjY6fD+E949vVzeZ7vHx8mM6uPCATi//DQAA//+MVAsnAgcAAA=="
 
-	tokendata, err := ParseToken(token)
+func TestParseToken(t *testing.T) {
+	tokendata, err := ParseToken(testJoinToken)
 	require.NoError(t, err)
 	require.Equal(t, "i6i3yg", tokendata.ID)
 	require.Equal(t, "https://172.17.0.2:6443", tokendata.URL)
 }
 
+// makeJoinToken builds a join token whose embedded kubeconfig points at server,
+// using the same encoding k0s uses, so tests don't depend on a hardcoded token.
+func makeJoinToken(t *testing.T, server string) string {
+	t.Helper()
+	kubeconfig := fmt.Sprintf(`apiVersion: v1
+kind: Config
+clusters:
+- name: k0s
+  cluster:
+    server: %s
+users:
+- name: kubelet-bootstrap
+  user:
+    token: testid.abcdefghijklmnop
+`, server)
+	token, err := encodeToken([]byte(kubeconfig))
+	require.NoError(t, err)
+	return token
+}
+
+func TestTokenWithJoinHost(t *testing.T) {
+	tokendata, err := ParseToken(makeJoinToken(t, "https://172.17.0.2:6443"))
+	require.NoError(t, err)
+	require.Equal(t, "testid", tokendata.ID)
+
+	rewritten, err := tokendata.WithJoinHost("10.0.0.1")
+	require.NoError(t, err)
+	require.Equal(t, "https://10.0.0.1:6443", rewritten.URL, "host is replaced, port preserved")
+	require.NotEqual(t, tokendata.Token, rewritten.Token, "token string is re-encoded")
+
+	// the re-encoded token must decode back to the rewritten URL while keeping
+	// the original token ID intact
+	reparsed, err := ParseToken(rewritten.Token)
+	require.NoError(t, err)
+	require.Equal(t, "https://10.0.0.1:6443", reparsed.URL)
+	require.Equal(t, tokendata.ID, reparsed.ID)
+}
+
 func TestUnmarshal(t *testing.T) {
 	t.Run("version given", func(t *testing.T) {
 		k0s := &K0s{}