diff --git a/pkg/domain/infosync/BUILD.bazel b/pkg/domain/infosync/BUILD.bazel index 1c6c935b65d17..a61c8a7cac2a2 100644 --- a/pkg/domain/infosync/BUILD.bazel +++ b/pkg/domain/infosync/BUILD.bazel @@ -72,7 +72,7 @@ go_test( ], embed = [":infosync"], flaky = True, - shard_count = 4, + shard_count = 5, deps = [ "//pkg/config/kerneltype", "//pkg/ddl/label", @@ -81,6 +81,7 @@ go_test( "//pkg/keyspace", "//pkg/meta/model", "//pkg/parser/ast", + "//pkg/sessionctx/vardef", "//pkg/testkit/testsetup", "@com_github_pingcap_failpoint//:failpoint", "@com_github_pingcap_kvproto//pkg/keyspacepb", diff --git a/pkg/domain/infosync/info.go b/pkg/domain/infosync/info.go index f64354bd6a6e7..d9b36bccb4208 100644 --- a/pkg/domain/infosync/info.go +++ b/pkg/domain/infosync/info.go @@ -83,6 +83,11 @@ const ( // ErrPrometheusAddrIsNotSet is the error that Prometheus address is not set in PD and etcd var ErrPrometheusAddrIsNotSet = dbterror.ClassDomain.NewStd(errno.ErrPrometheusAddrIsNotSet) +func init() { + vardef.SetClusterReadOnlyChecker(getClusterReadOnlyStatus) + vardef.SetReadOnlyStatusReporter(updateServerReadOnlyStatus) +} + // InfoSyncer stores server info to etcd when the tidb-server starts and delete when tidb-server shuts down. type InfoSyncer struct { // `etcdClient` must be used when keyspace is not set, or when the logic to each etcd path needs to be separated by keyspace. @@ -343,6 +348,47 @@ func GetAllServerInfo(ctx context.Context) (map[string]*serverinfo.ServerInfo, e return is.svrInfoSyncer.GetAllServerInfo(ctx) } +// UpdateServerReadOnlyStatus updates the local TiDB read-only status in the info syncer. +func UpdateServerReadOnlyStatus(ctx context.Context) error { + return updateServerReadOnlyStatus(ctx) +} + +func updateServerReadOnlyStatus(ctx context.Context) error { + is, err := getGlobalInfoSyncer() + if err != nil { + return nil + } + return is.svrInfoSyncer.UpdateServerReadOnlyStatus( + ctx, + vardef.RestrictedReadOnly.Load(), + vardef.VarTiDBSuperReadOnly.Load(), + ) +} + +func getClusterReadOnlyStatus(ctx context.Context) (bool, error) { + is, err := getGlobalInfoSyncer() + if err != nil { + return vardef.LocalTiDBReadOnlyStatus(), nil + } + allServerInfo, err := is.svrInfoSyncer.GetAllServerInfo(ctx) + if err != nil { + return false, err + } + return clusterReadOnlyStatusFromServerInfo(allServerInfo), nil +} + +func clusterReadOnlyStatusFromServerInfo(allServerInfo map[string]*serverinfo.ServerInfo) bool { + if len(allServerInfo) == 0 { + return false + } + for _, info := range allServerInfo { + if info == nil || !info.TiDBEffectiveReadOnly { + return false + } + } + return true +} + // UpdateServerLabel updates the server label for global info syncer. func UpdateServerLabel(ctx context.Context, labels map[string]string) error { is, err := getGlobalInfoSyncer() diff --git a/pkg/domain/infosync/info_test.go b/pkg/domain/infosync/info_test.go index aef07ce1e3fdd..31b1efc78be2b 100644 --- a/pkg/domain/infosync/info_test.go +++ b/pkg/domain/infosync/info_test.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tidb/pkg/keyspace" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/sessionctx/vardef" "github.com/pingcap/tidb/pkg/testkit/testsetup" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -210,3 +211,59 @@ func TestInfoSyncerMarshal(t *testing.T) { require.Equal(t, info.JSONServerID, decodeInfo.JSONServerID) require.Equal(t, info.Labels, decodeInfo.Labels) } + +func TestClusterReadOnlyStatus(t *testing.T) { + ctx := context.Background() + _, err := GlobalInfoSyncerInit(ctx, "test-read-only", func() uint64 { return 1 }, nil, nil, nil, nil, keyspace.CodecV1, false, nil) + require.NoError(t, err) + t.Cleanup(func() { + vardef.RestrictedReadOnly.Store(false) + vardef.VarTiDBSuperReadOnly.Store(false) + require.NoError(t, UpdateServerReadOnlyStatus(context.Background())) + }) + + vardef.RestrictedReadOnly.Store(false) + vardef.VarTiDBSuperReadOnly.Store(false) + require.NoError(t, UpdateServerReadOnlyStatus(ctx)) + on, err := vardef.GetClusterReadOnlyStatus(ctx) + require.NoError(t, err) + require.False(t, on) + + vardef.VarTiDBSuperReadOnly.Store(true) + require.NoError(t, UpdateServerReadOnlyStatus(ctx)) + on, err = vardef.GetClusterReadOnlyStatus(ctx) + require.NoError(t, err) + require.True(t, on) + + for _, tt := range []struct { + name string + infos map[string]*serverinfo.ServerInfo + expect bool + }{ + { + name: "all effective read-only", + infos: map[string]*serverinfo.ServerInfo{ + "tidb-a": {DynamicInfo: serverinfo.DynamicInfo{TiDBSuperReadOnly: true, TiDBEffectiveReadOnly: true}}, + "tidb-b": {DynamicInfo: serverinfo.DynamicInfo{TiDBRestrictedReadOnly: true, TiDBSuperReadOnly: true, TiDBEffectiveReadOnly: true}}, + }, + expect: true, + }, + { + name: "one instance is writable", + infos: map[string]*serverinfo.ServerInfo{ + "tidb-a": {DynamicInfo: serverinfo.DynamicInfo{TiDBSuperReadOnly: true, TiDBEffectiveReadOnly: true}}, + "tidb-b": {DynamicInfo: serverinfo.DynamicInfo{}}, + }, + expect: false, + }, + { + name: "no live instance", + infos: map[string]*serverinfo.ServerInfo{}, + expect: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expect, clusterReadOnlyStatusFromServerInfo(tt.infos)) + }) + } +} diff --git a/pkg/domain/serverinfo/BUILD.bazel b/pkg/domain/serverinfo/BUILD.bazel index 1457a8feeb199..bedd6acbe821b 100644 --- a/pkg/domain/serverinfo/BUILD.bazel +++ b/pkg/domain/serverinfo/BUILD.bazel @@ -34,7 +34,7 @@ go_test( srcs = ["syncer_test.go"], embed = [":serverinfo"], flaky = True, - shard_count = 2, + shard_count = 3, deps = [ "//pkg/config", "//pkg/config/kerneltype", diff --git a/pkg/domain/serverinfo/info.go b/pkg/domain/serverinfo/info.go index f4f0c2518c6d6..2a4d4bb0a0f48 100644 --- a/pkg/domain/serverinfo/info.go +++ b/pkg/domain/serverinfo/info.go @@ -88,13 +88,19 @@ func (i *StaticInfo) IsAssumed() bool { // To update the dynamic server information, use `InfoSyncer.cloneDynamicServerInfo` to obtain a copy of the dynamic server info. // After making modifications, use `InfoSyncer.setDynamicServerInfo` to update the dynamic server information. type DynamicInfo struct { - Labels map[string]string `json:"labels"` + Labels map[string]string `json:"labels"` + TiDBRestrictedReadOnly bool `json:"tidb_restricted_read_only,omitempty"` + TiDBSuperReadOnly bool `json:"tidb_super_read_only,omitempty"` + TiDBEffectiveReadOnly bool `json:"tidb_effective_read_only,omitempty"` } // Clone the DynamicInfo. func (d *DynamicInfo) Clone() *DynamicInfo { return &DynamicInfo{ - Labels: maps.Clone(d.Labels), + Labels: maps.Clone(d.Labels), + TiDBRestrictedReadOnly: d.TiDBRestrictedReadOnly, + TiDBSuperReadOnly: d.TiDBSuperReadOnly, + TiDBEffectiveReadOnly: d.TiDBEffectiveReadOnly, } } diff --git a/pkg/domain/serverinfo/syncer.go b/pkg/domain/serverinfo/syncer.go index b024685c33d14..d8e77a0bc9f47 100644 --- a/pkg/domain/serverinfo/syncer.go +++ b/pkg/domain/serverinfo/syncer.go @@ -180,6 +180,39 @@ func (s *Syncer) UpdateServerLabel(ctx context.Context, labels map[string]string return nil } +// UpdateServerReadOnlyStatus updates the local server read-only status in etcd. +func (s *Syncer) UpdateServerReadOnlyStatus(ctx context.Context, restricted, super bool) error { + dynamicInfo := s.cloneDynamicServerInfo() + effective := restricted || super + if dynamicInfo.TiDBRestrictedReadOnly == restricted && + dynamicInfo.TiDBSuperReadOnly == super && + dynamicInfo.TiDBEffectiveReadOnly == effective { + return nil + } + + dynamicInfo.TiDBRestrictedReadOnly = restricted + dynamicInfo.TiDBSuperReadOnly = super + dynamicInfo.TiDBEffectiveReadOnly = effective + if s.etcdCli == nil || s.session == nil { + s.setDynamicServerInfo(dynamicInfo) + return nil + } + + info := s.GetLocalServerInfo().Clone() + info.DynamicInfo = *dynamicInfo + infoBuf, err := info.Marshal() + if err != nil { + return errors.Trace(err) + } + str := string(hack.String(infoBuf)) + err = util.PutKVToEtcd(ctx, s.etcdCli, KeyOpDefaultRetryCnt, s.serverInfoPath, str, clientv3.WithLease(s.session.Lease())) + if err != nil { + return err + } + s.setDynamicServerInfo(dynamicInfo) + return nil +} + // cloneDynamicServerInfo returns a clone of the dynamic server info. func (s *Syncer) cloneDynamicServerInfo() *DynamicInfo { return s.info.Load().DynamicInfo.Clone() @@ -204,8 +237,12 @@ func (s *Syncer) GetAllServerInfo(ctx context.Context) (map[string]*ServerInfo, }) allInfo := make(map[string]*ServerInfo) if s.etcdCli == nil { - info := s.info.Load() - allInfo[info.ID] = getServerInfo(info.ID, info.ServerIDGetter, "") + localInfo := s.info.Load() + info := getServerInfo(localInfo.ID, localInfo.ServerIDGetter, localInfo.AssumedKeyspace) + info.TiDBRestrictedReadOnly = localInfo.TiDBRestrictedReadOnly + info.TiDBSuperReadOnly = localInfo.TiDBSuperReadOnly + info.TiDBEffectiveReadOnly = localInfo.TiDBEffectiveReadOnly + allInfo[info.ID] = info return allInfo, nil } allInfo, err := getInfo(ctx, s.etcdCli, ServerInformationPath, KeyOpDefaultRetryCnt, KeyOpDefaultTimeout, clientv3.WithPrefix()) diff --git a/pkg/domain/serverinfo/syncer_test.go b/pkg/domain/serverinfo/syncer_test.go index 4ccd02a1965e1..39627e98557b4 100644 --- a/pkg/domain/serverinfo/syncer_test.go +++ b/pkg/domain/serverinfo/syncer_test.go @@ -182,3 +182,32 @@ func TestAssumedServerInfoSyncer(t *testing.T) { require.Equal(t, "ks1", info.AssumedKeyspace) require.EqualValues(t, keyspace.System, info.Keyspace) } + +func TestGetAllServerInfoWithoutEtcd(t *testing.T) { + bak := config.GetGlobalConfig() + t.Cleanup(func() { + config.StoreGlobalConfig(bak) + }) + config.UpdateGlobal(func(conf *config.Config) { + conf.Status.StatusPort = 10080 + conf.Labels = map[string]string{"old": "label"} + }) + + syncer := NewSyncer("1", func() uint64 { return 1 }, nil, nil) + require.NoError(t, syncer.UpdateServerReadOnlyStatus(context.Background(), true, false)) + + config.UpdateGlobal(func(conf *config.Config) { + conf.Status.StatusPort = 12345 + conf.Labels = map[string]string{"fresh": "label"} + }) + + infos, err := syncer.GetAllServerInfo(context.Background()) + require.NoError(t, err) + info := infos["1"] + require.NotNil(t, info) + require.EqualValues(t, 12345, info.StatusPort) + require.Equal(t, map[string]string{"fresh": "label"}, info.Labels) + require.True(t, info.TiDBRestrictedReadOnly) + require.False(t, info.TiDBSuperReadOnly) + require.True(t, info.TiDBEffectiveReadOnly) +} diff --git a/pkg/sessionctx/vardef/BUILD.bazel b/pkg/sessionctx/vardef/BUILD.bazel index 84699330e8b71..e2ca169958de7 100644 --- a/pkg/sessionctx/vardef/BUILD.bazel +++ b/pkg/sessionctx/vardef/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "vardef", srcs = [ + "readonly.go", "runtime.go", "sysvar.go", "tidb_vars.go", diff --git a/pkg/sessionctx/vardef/readonly.go b/pkg/sessionctx/vardef/readonly.go new file mode 100644 index 0000000000000..f5ee7b845d1fe --- /dev/null +++ b/pkg/sessionctx/vardef/readonly.go @@ -0,0 +1,64 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vardef + +import ( + "context" + "sync/atomic" +) + +// ClusterReadOnlyChecker returns whether all live TiDB instances are effectively read-only. +type ClusterReadOnlyChecker func(context.Context) (bool, error) + +// ReadOnlyStatusReporter publishes the local TiDB read-only status. +type ReadOnlyStatusReporter func(context.Context) error + +var ( + clusterReadOnlyChecker atomic.Value + readOnlyStatusReporter atomic.Value +) + +// LocalTiDBReadOnlyStatus returns the local TiDB effective read-only status. +func LocalTiDBReadOnlyStatus() bool { + return RestrictedReadOnly.Load() || VarTiDBSuperReadOnly.Load() +} + +// SetClusterReadOnlyChecker registers the checker used by the derived tidb_is_read_only variable. +func SetClusterReadOnlyChecker(checker ClusterReadOnlyChecker) { + clusterReadOnlyChecker.Store(checker) +} + +// GetClusterReadOnlyStatus returns whether the whole TiDB cluster is effectively read-only. +func GetClusterReadOnlyStatus(ctx context.Context) (bool, error) { + checker, ok := clusterReadOnlyChecker.Load().(ClusterReadOnlyChecker) + if !ok || checker == nil { + return LocalTiDBReadOnlyStatus(), nil + } + return checker(ctx) +} + +// SetReadOnlyStatusReporter registers a reporter for the local TiDB read-only status. +func SetReadOnlyStatusReporter(reporter ReadOnlyStatusReporter) { + readOnlyStatusReporter.Store(reporter) +} + +// ReportReadOnlyStatus publishes the local TiDB read-only status when a reporter is available. +func ReportReadOnlyStatus(ctx context.Context) error { + reporter, ok := readOnlyStatusReporter.Load().(ReadOnlyStatusReporter) + if !ok || reporter == nil { + return nil + } + return reporter(ctx) +} diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index 13807b903f201..ee5adec737b53 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -780,6 +780,9 @@ const ( // TiDBSuperReadOnly is tidb's variant of mysql's super_read_only, which has some differences from mysql's super_read_only. TiDBSuperReadOnly = "tidb_super_read_only" + // TiDBIsReadOnly is a read-only derived status that indicates whether all TiDB instances are effectively read-only. + TiDBIsReadOnly = "tidb_is_read_only" + // TiDBShardAllocateStep indicates the max size of continuous rowid shard in one transaction. TiDBShardAllocateStep = "tidb_shard_allocate_step" // TiDBEnableTelemetry indicates that whether usage data report to PingCAP is enabled. diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 98780f1406ea8..3674821c99a12 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -81,6 +81,12 @@ func withMinValue(minVal int64) execConcurrencySysVarOption { return func(sv *SysVar) { sv.MinValue = minVal } } +func reportTiDBReadOnlyStatus(ctx context.Context) { + if err := vardef.ReportReadOnlyStatus(ctx); err != nil { + logutil.BgLogger().Warn("update TiDB read-only status failed", zap.Error(err)) + } +} + // newExecConcurrencySysVar creates a session/global SysVar for executor concurrency settings. func newExecConcurrencySysVar(name string, defValue int, setter concurrencySetter, opts ...execConcurrencySysVarOption) *SysVar { sv := &SysVar{ @@ -971,7 +977,7 @@ var defaultSysVars = []*SysVar{ tikvstore.TxnCommitBatchSize.Store(uint64(TidbOptInt64(val, int64(tikvstore.DefTxnCommitBatchSize)))) return nil }}, - {Scope: vardef.ScopeGlobal, Name: vardef.TiDBRestrictedReadOnly, Value: BoolToOnOff(vardef.DefTiDBRestrictedReadOnly), Type: vardef.TypeBool, SetGlobal: func(_ context.Context, s *SessionVars, val string) error { + {Scope: vardef.ScopeGlobal, Name: vardef.TiDBRestrictedReadOnly, Value: BoolToOnOff(vardef.DefTiDBRestrictedReadOnly), Type: vardef.TypeBool, SetGlobal: func(ctx context.Context, s *SessionVars, val string) error { on := TiDBOptOn(val) // For user initiated SET GLOBAL, also change the value of TiDBSuperReadOnly if on && s.StmtCtx.StmtType == "Set" { @@ -985,6 +991,7 @@ var defaultSysVars = []*SysVar{ } } vardef.RestrictedReadOnly.Store(on) + reportTiDBReadOnlyStatus(ctx) return nil }}, {Scope: vardef.ScopeGlobal, Name: vardef.TiDBSuperReadOnly, Value: BoolToOnOff(vardef.DefTiDBSuperReadOnly), Type: vardef.TypeBool, Validation: func(s *SessionVars, normalizedValue string, _ string, _ vardef.ScopeFlag) (string, error) { @@ -999,10 +1006,18 @@ var defaultSysVars = []*SysVar{ } } return normalizedValue, nil - }, SetGlobal: func(_ context.Context, s *SessionVars, val string) error { + }, SetGlobal: func(ctx context.Context, s *SessionVars, val string) error { vardef.VarTiDBSuperReadOnly.Store(TiDBOptOn(val)) + reportTiDBReadOnlyStatus(ctx) return nil }}, + {Scope: vardef.ScopeGlobal, Name: vardef.TiDBIsReadOnly, Value: vardef.Off, Type: vardef.TypeBool, ReadOnly: true, GetGlobal: func(ctx context.Context, s *SessionVars) (string, error) { + on, err := vardef.GetClusterReadOnlyStatus(ctx) + if err != nil { + return vardef.Off, err + } + return BoolToOnOff(on), nil + }}, {Scope: vardef.ScopeGlobal, Name: vardef.TiDBEnableGOGCTuner, Value: BoolToOnOff(vardef.DefTiDBEnableGOGCTuner), Type: vardef.TypeBool, SetGlobal: func(_ context.Context, s *SessionVars, val string) error { on := TiDBOptOn(val) gctuner.EnableGOGCTuner.Store(on) diff --git a/pkg/sessionctx/variable/sysvar_test.go b/pkg/sessionctx/variable/sysvar_test.go index 25c907a2d494b..9203c34f8eadd 100644 --- a/pkg/sessionctx/variable/sysvar_test.go +++ b/pkg/sessionctx/variable/sysvar_test.go @@ -445,6 +445,40 @@ func TestReadOnlyNoop(t *testing.T) { } } +func TestTiDBIsReadOnly(t *testing.T) { + vars := NewSessionVars(nil) + sv := GetSysVar(vardef.TiDBIsReadOnly) + require.NotNil(t, sv) + require.True(t, sv.ReadOnly) + require.True(t, sv.HasGlobalScope()) + + t.Cleanup(func() { + vardef.RestrictedReadOnly.Store(false) + vardef.VarTiDBSuperReadOnly.Store(false) + }) + + _, err := sv.Validate(vars, vardef.On, vardef.ScopeGlobal) + require.Error(t, err) + require.Equal(t, "[variable:1238]Variable 'tidb_is_read_only' is a read only variable", err.Error()) + + vardef.RestrictedReadOnly.Store(false) + vardef.VarTiDBSuperReadOnly.Store(false) + val, err := sv.GetGlobalFromHook(context.Background(), vars) + require.NoError(t, err) + require.Equal(t, vardef.Off, val) + + vardef.VarTiDBSuperReadOnly.Store(true) + val, err = sv.GetGlobalFromHook(context.Background(), vars) + require.NoError(t, err) + require.Equal(t, vardef.On, val) + + vardef.VarTiDBSuperReadOnly.Store(false) + vardef.RestrictedReadOnly.Store(true) + val, err = sv.GetGlobalFromHook(context.Background(), vars) + require.NoError(t, err) + require.Equal(t, vardef.On, val) +} + func TestSkipInit(t *testing.T) { sv := SysVar{Scope: vardef.ScopeGlobal, Name: "skipinit1", Value: vardef.On, Type: vardef.TypeBool} require.True(t, sv.SkipInit())