diff --git a/phase/install_controllers.go b/phase/install_controllers.go index 8a2567a8a..079845b30 100644 --- a/phase/install_controllers.go +++ b/phase/install_controllers.go @@ -129,16 +129,40 @@ func (p *InstallControllers) Run(ctx context.Context) error { h.Metadata.K0sTokenData.URL = p.Config.Spec.KubeAPIURL() } } + // The freshly initialized leader is the only controller guaranteed to be up + // and reachable during this phase, so it is the safe fallback join target if + // the token-embedded URL (derived from spec.api.externalAddress) cannot be + // reached - e.g. an externally managed keepalived VIP not currently assigned + // to the leader. + leaderJoinHost := p.leader.PrivateAddress + if leaderJoinHost == "" { + leaderJoinHost = p.leader.Address() + } + err := p.parallelDo(ctx, p.hosts, func(_ context.Context, h *cluster.Host) error { - if p.IsWet() || !p.leader.Metadata.DryRunFakeLeader { - log.Infof("%s: validating api connection to %s", h, h.Metadata.K0sTokenData.URL) - if err := retry.WithDefaultTimeout(ctx, node.HTTPStatusFunc(h, h.Metadata.K0sTokenData.URL, 200, 401, 404)); err != nil { + if !p.IsWet() && p.leader.Metadata.DryRunFakeLeader { + log.Warnf("%s: dry-run: skipping api connection validation because cluster is not actually running", h) + return nil + } + + log.Infof("%s: validating api connection to %s", h, h.Metadata.K0sTokenData.URL) + if err := retry.WithDefaultTimeout(ctx, node.HTTPStatusFunc(h, h.Metadata.K0sTokenData.URL, 200, 401, 404)); err == nil { + return nil + } else { + // Fall back to joining via the leader's own address. Rewrite the + // token so the actual join targets the leader too, not just this + // preflight check. + rewritten, rerr := h.Metadata.K0sTokenData.WithJoinHost(leaderJoinHost) + if rerr != nil { return fmt.Errorf("failed to connect from controller to kubernetes api - check networking: %w", err) } - } else { - log.Warnf("%s: dry-run: skipping api connection validation to because cluster is not actually running", h) + log.Warnf("%s: api connection to %s failed, retrying via leader address %s", h, h.Metadata.K0sTokenData.URL, rewritten.URL) + if rerr := retry.WithDefaultTimeout(ctx, node.HTTPStatusFunc(h, rewritten.URL, 200, 401, 404)); rerr != nil { + return fmt.Errorf("failed to connect from controller to kubernetes api - check networking: %w", err) + } + h.Metadata.K0sTokenData = rewritten + return nil } - return nil }) if err != nil { return err diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go index 9128ac9ba..49925e46f 100644 --- a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go +++ b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s.go @@ -1,11 +1,14 @@ package cluster import ( + "bytes" "compress/gzip" "context" "encoding/base64" "fmt" "io" + "net" + "net/url" "strings" "time" @@ -181,6 +184,88 @@ type TokenData struct { Kubeconfig []byte } +// WithJoinHost returns a copy of the token data with the embedded kubeconfig's +// cluster server host replaced by host (scheme and port are preserved) and the +// token re-encoded. Because an actual controller/worker join reads the server +// URL from the token file, rewriting the host here redirects the join itself, +// not just k0sctl's preflight validation. +// +// This is used to make joins target a known-reachable address (the freshly +// initialized leader) when the token-embedded address derived from +// spec.api.externalAddress is not reachable during bootstrap, e.g. an +// externally managed keepalived VIP that is not currently assigned to the +// leader. +// +// Note: the leader's serving certificate must include host in its SANs or the +// join's TLS verification will fail. +func (d TokenData) WithJoinHost(host string) (TokenData, error) { + if len(d.Kubeconfig) == 0 { + return d, fmt.Errorf("token has no kubeconfig data to rewrite") + } + + cfg := dig.Mapping{} + if err := yaml.Unmarshal(d.Kubeconfig, &cfg); err != nil { + return d, fmt.Errorf("failed to unmarshal token: %w", err) + } + + clusters, ok := cfg.Dig("clusters").([]any) + if !ok || len(clusters) < 1 { + return d, fmt.Errorf("failed to find clusters in token") + } + cluster, ok := clusters[0].(dig.Mapping) + if !ok { + return d, fmt.Errorf("failed to find cluster in token") + } + clusterData, ok := cluster.Dig("cluster").(dig.Mapping) + if !ok { + return d, fmt.Errorf("failed to find cluster data in token") + } + server, ok := clusterData["server"].(string) + if !ok || server == "" { + return d, fmt.Errorf("failed to find cluster url in token") + } + + u, err := url.Parse(server) + if err != nil { + return d, fmt.Errorf("failed to parse cluster url %q: %w", server, err) + } + if port := u.Port(); port != "" { + u.Host = net.JoinHostPort(host, port) + } else { + u.Host = host + } + clusterData["server"] = u.String() + + kubeconfig, err := yaml.Marshal(cfg) + if err != nil { + return d, fmt.Errorf("failed to marshal token: %w", err) + } + token, err := encodeToken(kubeconfig) + if err != nil { + return d, err + } + + return TokenData{ID: d.ID, URL: u.String(), Token: token, Kubeconfig: kubeconfig}, nil +} + +// encodeToken encodes a kubeconfig into a k0s join token (gzip + base64), the +// inverse of the decoding performed by ParseToken. +func encodeToken(kubeconfig []byte) (string, error) { + var buf bytes.Buffer + b64 := base64.NewEncoder(base64.StdEncoding, &buf) + gz := gzip.NewWriter(b64) + if _, err := gz.Write(kubeconfig); err != nil { + return "", fmt.Errorf("failed to compress token: %w", err) + } + if err := gz.Close(); err != nil { + return "", fmt.Errorf("failed to compress token: %w", err) + } + if err := b64.Close(); err != nil { + return "", fmt.Errorf("failed to encode token: %w", err) + } + return buf.String(), nil +} + // ParseToken returns TokenData for a token string func ParseToken(s string) (TokenData, error) { data := TokenData{Token: s} diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s_test.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s_test.go index 17f42058a..f4d9c3500 100644 --- a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s_test.go +++ b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/k0s_test.go @@ -1,6 +1,7 @@ package cluster import ( + "fmt" "testing" "github.com/creasty/defaults" @@ -10,15 +11,55 @@ import ( "gopkg.in/yaml.v2" ) -func TestParseToken(t *testing.T) { - token := "H4sIAAAAAAAC/2xVXY/iOBZ9r1/BH6geO4GeAWkfKiEmGGLKjn1N/BbidAFOgjuk+Frtf18V3SPtSvN2fc/ROdaVfc9L6Q9Q9+fDqZuNLvilaj7PQ92fZy+vo9/17GU0Go3OdX+p+9loPwz+PPvjD/xn8A3/+Q19C2bfx+Pwyanqfjj8OFTlUL+Wn8P+1B+G+6sth3I2WudoWOc4FspSeYjmAqjKlaEcESWeGBpih2muRCQSNucavEEkzBWNDGoApDV1t19W6uNSbJsyRzS1mPc7TVdiDknV0qNFQmjl1zvsaZmao3RECHVd8YZEFtlEgGW8ISmXBIQiY6km+wwbr5v9yoIvVHs71pL81CAio0yYpQ2DJMFSe1InWHEZMZHQveiqa/3hf2Eg+v/FpKJdnZifHCA2aKK5IwwSsbVzYnZgJkWLdUZ8IbfCZA5CE1hSKhxliZ2rkKRxw2hxZIlSEHMgwFWCckUTi8iTmyNy+ZqJUtktO2Y9C8Wpuk8DsTUT7ehnjt9uBTQ0T7yDB9nyw+A4Tlb5wt2NbHgB5LSJpwvR2Ytpp6oKm/lG2ZvUZoDERjs9vubzamxJcZEaX6vDwLKWFeUWIoOqi7z/hWx7c2q77DfcJ5BkQQFAyxYw6xix8BZILAar8Ha3GM7l420ssZ/UZE/rrQtUytSus4ssXGKOissKkdgiOskw1fowPKRqxnFLPy0hj1pPvV6IC0t4AOhGgZDlZjFdGYdXLBVZBozKrUccW6Ra2mQNm5sF9bsHXRVqv8lB7E3XmNyZjKHTSm7Jp82HyxoJDom56HY8zgFa6/xCoOtdIL8qF8t71rDUYBZAI247ZHnpiluZn+9WNu8GsvEusFuOpvNS20J/+GUN1aN2U2kfpFQouVaBj3PsW6VgXwXVeJfSd4DlLdN2JR+gqoAed8hEBcB7OXc4J3Dl2jLuSCQCL0pHo9jhiCU2ygCcSC3hh2moFEQWNTFvfaQS2snGLJXDMdfFWCiquBKRUh8XqZZXgZIbaJEYTLbcUQnBtLDkY8VbWuzmMAhH97ka1tWWKN1lvQFLICEb3tq+0vu+VNXEPqKvN/gQjkQSsejLv3BsUjTRNk8mpNbMF46d1Ju/SURPRWihBOJtS5eVwp9ZQhvIB8+UCo1ksSXg7IPcS2wNc35cphHKVKNE4rebbSR2ODpxd5uYAA/VfH+JW9Jt1GRv231eJ9mj1uao2+Z7pRrB2ulP4+xF5kOxDtUF3PLKJXmXCb4XgQmzuRFVmmGZnCaA/nrIBdCvuRduvMpVs8lcNi7UcDVhRG0A93JLYpP66yqYgJoLoZumlQ9x2xFD8znIkux77oacdWqSdZSVyjCWnkKmb+9WDz/Nh5+b9O1SIDIUHaC6bW5V4qFsYSnSRmUIloXCuV1MaE7IsQAxBkR5ndqASRZtFDVGm7VszHGzwEfhJqzUzTV2tMi1iG369dfsmjVvkxKKfhMPgjsccEUPLMmCTcJCsTDrfGHGdXsOJcBpo4ezQd7sQroC3EQrdLtVD+Z16lZCY58rEO8SrX7vZiId/+AIckiaRa5YBIl67uU1P/3rZTTqyraejRw6v1Snbqhvw6+U+FX/Som/I+PJ+mp8np+nz13d1MPr7nQazkNf+v9X++z7uhte/1Z6Nt2hs7NRfOp+HD5efF//qPu6q+rzbPTv/7x8qT7Nf4v8g/zT+HmF4eTqbjY6fD+E949vVzeZ7vHx8mM6uPCATi//DQAA//+MVAsnAgcAAA==" +// testJoinToken is a sample (expired, non-secret) k0s join token fixture whose +// embedded kubeconfig points at https://172.17.0.2:6443 with token ID i6i3yg. +const testJoinToken = "H4sIAAAAAAAC/2xVXY/iOBZ9r1/BH6geO4GeAWkfKiEmGGLKjn1N/BbidAFOgjuk+Frtf18V3SPtSvN2fc/ROdaVfc9L6Q9Q9+fDqZuNLvilaj7PQ92fZy+vo9/17GU0Go3OdX+p+9loPwz+PPvjD/xn8A3/+Q19C2bfx+Pwyanqfjj8OFTlUL+Wn8P+1B+G+6sth3I2WudoWOc4FspSeYjmAqjKlaEcESWeGBpih2muRCQSNucavEEkzBWNDGoApDV1t19W6uNSbJsyRzS1mPc7TVdiDknV0qNFQmjl1zvsaZmao3RECHVd8YZEFtlEgGW8ISmXBIQiY6km+wwbr5v9yoIvVHs71pL81CAio0yYpQ2DJMFSe1InWHEZMZHQveiqa/3hf2Eg+v/FpKJdnZifHCA2aKK5IwwSsbVzYnZgJkWLdUZ8IbfCZA5CE1hSKhxliZ2rkKRxw2hxZIlSEHMgwFWCckUTi8iTmyNy+ZqJUtktO2Y9C8Wpuk8DsTUT7ehnjt9uBTQ0T7yDB9nyw+A4Tlb5wt2NbHgB5LSJpwvR2Ytpp6oKm/lG2ZvUZoDERjs9vubzamxJcZEaX6vDwLKWFeUWIoOqi7z/hWx7c2q77DfcJ5BkQQFAyxYw6xix8BZILAar8Ha3GM7l420ssZ/UZE/rrQtUytSus4ssXGKOissKkdgiOskw1fowPKRqxnFLPy0hj1pPvV6IC0t4AOhGgZDlZjFdGYdXLBVZBozKrUccW6Ra2mQNm5sF9bsHXRVqv8lB7E3XmNyZjKHTSm7Jp82HyxoJDom56HY8zgFa6/xCoOtdIL8qF8t71rDUYBZAI247ZHnpiluZn+9WNu8GsvEusFuOpvNS20J/+GUN1aN2U2kfpFQouVaBj3PsW6VgXwXVeJfSd4DlLdN2JR+gqoAed8hEBcB7OXc4J3Dl2jLuSCQCL0pHo9jhiCU2ygCcSC3hh2moFEQWNTFvfaQS2snGLJXDMdfFWCiquBKRUh8XqZZXgZIbaJEYTLbcUQnBtLDkY8VbWuzmMAhH97ka1tWWKN1lvQFLICEb3tq+0vu+VNXEPqKvN/gQjkQSsejLv3BsUjTRNk8mpNbMF46d1Ju/SURPRWihBOJtS5eVwp9ZQhvIB8+UCo1ksSXg7IPcS2wNc35cphHKVKNE4rebbSR2ODpxd5uYAA/VfH+JW9Jt1GRv231eJ9mj1uao2+Z7pRrB2ulP4+xF5kOxDtUF3PLKJXmXCb4XgQmzuRFVmmGZnCaA/nrIBdCvuRduvMpVs8lcNi7UcDVhRG0A93JLYpP66yqYgJoLoZumlQ9x2xFD8znIkux77oacdWqSdZSVyjCWnkKmb+9WDz/Nh5+b9O1SIDIUHaC6bW5V4qFsYSnSRmUIloXCuV1MaE7IsQAxBkR5ndqASRZtFDVGm7VszHGzwEfhJqzUzTV2tMi1iG369dfsmjVvkxKKfhMPgjsccEUPLMmCTcJCsTDrfGHGdXsOJcBpo4ezQd7sQroC3EQrdLtVD+Z16lZCY58rEO8SrX7vZiId/+AIckiaRa5YBIl67uU1P/3rZTTqyraejRw6v1Snbqhvw6+U+FX/Som/I+PJ+mp8np+nz13d1MPr7nQazkNf+v9X++z7uhte/1Z6Nt2hs7NRfOp+HD5efF//qPu6q+rzbPTv/7x8qT7Nf4v8g/zT+HmF4eTqbjY6fD+E949vVzeZ7vHx8mM6uPCATi//DQAA//+MVAsnAgcAAA==" - tokendata, err := ParseToken(token) +func TestParseToken(t *testing.T) { + tokendata, err := ParseToken(testJoinToken) require.NoError(t, err) require.Equal(t, "i6i3yg", tokendata.ID) require.Equal(t, "https://172.17.0.2:6443", tokendata.URL) } +// makeJoinToken builds a join token whose embedded kubeconfig points at server, +// using the same encoding k0s uses, so tests don't depend on a hardcoded token. +func makeJoinToken(t *testing.T, server string) string { + t.Helper() + kubeconfig := fmt.Sprintf(`apiVersion: v1 +kind: Config +clusters: +- name: k0s + cluster: + server: %s +users: +- name: kubelet-bootstrap + user: + token: testid.abcdefghijklmnop +`, server) + token, err := encodeToken([]byte(kubeconfig)) + require.NoError(t, err) + return token +} + +func TestTokenWithJoinHost(t *testing.T) { + tokendata, err := ParseToken(makeJoinToken(t, "https://172.17.0.2:6443")) + require.NoError(t, err) + require.Equal(t, "testid", tokendata.ID) + + rewritten, err := tokendata.WithJoinHost("10.0.0.1") + require.NoError(t, err) + require.Equal(t, "https://10.0.0.1:6443", rewritten.URL, "host is replaced, port preserved") + require.NotEqual(t, tokendata.Token, rewritten.Token, "token string is re-encoded") + + // the re-encoded token must decode back to the rewritten URL while keeping + // the original token ID intact + reparsed, err := ParseToken(rewritten.Token) + require.NoError(t, err) + require.Equal(t, "https://10.0.0.1:6443", reparsed.URL) + require.Equal(t, tokendata.ID, reparsed.ID) +} + func TestUnmarshal(t *testing.T) { t.Run("version given", func(t *testing.T) { k0s := &K0s{}