From 252c474b8ee80554a6500bbe9482517e3ac10f0d Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Wed, 3 Jun 2026 14:17:58 +0800 Subject: [PATCH 1/8] try fix Signed-off-by: JaySon-Huang --- contrib/tiflash-columnar-hub/Cargo.lock | 191 ++++++------------ contrib/tiflash-columnar-hub/Cargo.toml | 11 +- .../hub-runtime/Cargo.toml | 6 +- 3 files changed, 74 insertions(+), 134 deletions(-) diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 7016b63954f..9810d9d97f0 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -14,7 +14,7 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ - "gimli 0.32.3", + "gimli", ] [[package]] @@ -173,7 +173,7 @@ dependencies = [ "bytes", "codec", "engine_traits", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "match-template", "thiserror 1.0.69", "tikv_alloc", @@ -382,7 +382,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "hyper-tls 0.5.0", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "md5", "prometheus", @@ -952,7 +952,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object 0.37.3", + "object", "rustc-demangle", "windows-link", ] @@ -1120,7 +1120,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "kvengine", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "pd_client", "rand 0.8.6", "serde", @@ -1396,7 +1396,7 @@ dependencies = [ "charabia", "fastrand 2.4.1", "lazy_static", - "memmap2 0.9.10", + "memmap2", "once_cell", "ordered-float 5.3.0", "prost 0.13.5", @@ -1404,7 +1404,7 @@ dependencies = [ "stable_deref_trait", "tantivy", "tempfile", - "tipb", + "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", "workspace-hack", ] @@ -1417,7 +1417,7 @@ dependencies = [ "derive_more", "error_code", "futures-io", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "openssl", "prometheus", @@ -1443,7 +1443,7 @@ dependencies = [ "fxhash", "hmac 0.10.1", "kvenginepb", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log", "openssl", @@ -1570,15 +1570,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "cpp_demangle" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" -dependencies = [ - "cfg-if", -] - [[package]] name = "cpu-time" version = "1.0.0" @@ -2112,7 +2103,7 @@ dependencies = [ "futures 0.3.32", "futures-util", "hex 0.4.3", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "online_config", "openssl", @@ -2140,7 +2131,7 @@ dependencies = [ "error_code", "fail 0.5.1", "file_system", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "log_wrappers", "protobuf 2.8.0", "quick_cache", @@ -2230,7 +2221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2239,7 +2230,7 @@ version = "0.0.1" source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "grpcio", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "raft", "serde", @@ -2314,12 +2305,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - [[package]] name = "farmhash" version = "1.1.5" @@ -2462,20 +2447,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "framehop" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a5a3f0acb82df800ca3aa50c0d60d286c5d13d4cfc3114b3a9663f13b032fe" -dependencies = [ - "arrayvec", - "cfg-if", - "fallible-iterator 0.3.0", - "gimli 0.31.1", - "macho-unwind-info", - "pe-unwind-info", -] - [[package]] name = "fs2" version = "0.4.3" @@ -2720,16 +2691,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" -dependencies = [ - "fallible-iterator 0.3.0", - "stable_deref_trait", -] - [[package]] name = "gimli" version = "0.32.3" @@ -2928,7 +2889,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f530c66081961311028f67d22da8e3f739dab6979fc200f199c3ef0fb2e194a" dependencies = [ - "fallible-iterator 0.2.0", + "fallible-iterator", ] [[package]] @@ -3363,7 +3324,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3466,7 +3427,7 @@ version = "0.1.0" source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "byteorder", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "log_wrappers", "thiserror 1.0.69", "tikv_alloc", @@ -3527,13 +3488,13 @@ dependencies = [ "itertools 0.10.5", "keys", "kvenginepb", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "libc", "log_wrappers", "lz4", "maybe-async", - "memmap2 0.9.10", + "memmap2", "nix 0.24.3", "num_cpus", "online_config", @@ -3570,7 +3531,7 @@ dependencies = [ "tikv_alloc", "tikv_util", "time 0.1.45", - "tipb", + "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", "tokio", "tokio-util", "tracker", @@ -3595,7 +3556,20 @@ dependencies = [ [[package]] name = "kvproto" version = "0.0.2" -source = "git+ssh://git@github.com/pingcap/kvproto.git?rev=02dc8a9dc01827367f9d91ff2d88c447d3a838ab#02dc8a9dc01827367f9d91ff2d88c447d3a838ab" +source = "git+https://github.com/pingcap/kvproto.git?rev=02dc8a9dc01827367f9d91ff2d88c447d3a838ab#02dc8a9dc01827367f9d91ff2d88c447d3a838ab" +dependencies = [ + "bytes", + "futures 0.3.32", + "grpcio", + "protobuf 2.8.0", + "protobuf-build", + "raft-proto", +] + +[[package]] +name = "kvproto" +version = "0.0.2" +source = "git+https://github.com/pingcap/kvproto.git#1e6ff1c80df24a9ead2b4408133af23c09273718" dependencies = [ "bytes", "futures 0.3.32", @@ -3928,17 +3902,6 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" -[[package]] -name = "macho-unwind-info" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149" -dependencies = [ - "thiserror 2.0.18", - "zerocopy", - "zerocopy-derive", -] - [[package]] name = "match-template" version = "0.0.1" @@ -4018,15 +3981,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - [[package]] name = "memmap2" version = "0.9.10" @@ -4346,16 +4300,6 @@ dependencies = [ "url", ] -[[package]] -name = "object" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" -dependencies = [ - "flate2", - "memchr", -] - [[package]] name = "object" version = "0.37.3" @@ -4607,7 +4551,7 @@ dependencies = [ "grpcio", "hex 0.4.3", "http 0.2.12", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log", "log_wrappers", @@ -4632,19 +4576,6 @@ dependencies = [ "yatp", ] -[[package]] -name = "pe-unwind-info" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59" -dependencies = [ - "arrayvec", - "bitflags 2.11.1", - "thiserror 2.0.18", - "zerocopy", - "zerocopy-derive", -] - [[package]] name = "peeking_take_while" version = "0.1.2" @@ -4762,19 +4693,17 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pprof" version = "0.15.0" -source = "git+https://github.com/tikv/pprof-rs.git?rev=01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8#01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" dependencies = [ "aligned-vec", "backtrace", "cfg-if", "findshlibs", - "framehop", "inferno 0.11.21", "libc", "log", - "memmap2 0.5.10", "nix 0.26.4", - "object 0.29.0", "once_cell", "protobuf 3.7.2", "protobuf-codegen 3.7.2", @@ -5805,7 +5734,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5818,7 +5747,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5934,7 +5863,7 @@ dependencies = [ "slog-term", "tidb_query_datatype", "tikv_util", - "tipb", + "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", "txn_types", "workspace-hack", ] @@ -5979,7 +5908,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "hyper-rustls", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "prometheus", "rustls", @@ -6039,7 +5968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6498,7 +6427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" dependencies = [ "debugid", - "memmap2 0.9.10", + "memmap2", "stable_deref_trait", "uuid 1.23.1", ] @@ -6509,7 +6438,6 @@ version = "12.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" dependencies = [ - "cpp_demangle", "rustc-demangle", "symbolic-common", ] @@ -6607,7 +6535,7 @@ dependencies = [ "lru", "lz4_flex", "measure_time", - "memmap2 0.9.10", + "memmap2", "num_cpus", "once_cell", "oneshot", @@ -6737,7 +6665,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6850,7 +6778,7 @@ dependencies = [ "derive_more", "error_code", "futures 0.3.32", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log_wrappers", "maybe-async", @@ -6886,7 +6814,7 @@ dependencies = [ "encoding_rs 0.8.29", "error_code", "hex 0.4.3", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "log_wrappers", "match-template", @@ -6907,7 +6835,7 @@ dependencies = [ "tidb_query_common", "tikv_alloc", "tikv_util", - "tipb", + "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", "workspace-hack", ] @@ -6930,7 +6858,7 @@ dependencies = [ "keys", "kvengine", "kvenginepb", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git?rev=02dc8a9dc01827367f9d91ff2d88c447d3a838ab)", "lazy_static", "num_cpus", "pd_client", @@ -6948,7 +6876,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tikv_util", - "tipb", + "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git?rev=9fd33ded6f2c94633e601104cde26f2bdb5147d5)", "tokio", "tokio-executor", "toml", @@ -7018,7 +6946,7 @@ dependencies = [ "grpcio", "http 0.2.12", "hyper 0.14.26", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "libc", "log", @@ -7122,7 +7050,18 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tipb" version = "0.0.1" -source = "git+ssh://git@github.com/pingcap/tipb.git?rev=9fd33ded6f2c94633e601104cde26f2bdb5147d5#9fd33ded6f2c94633e601104cde26f2bdb5147d5" +source = "git+https://github.com/pingcap/tipb.git?rev=9fd33ded6f2c94633e601104cde26f2bdb5147d5#9fd33ded6f2c94633e601104cde26f2bdb5147d5" +dependencies = [ + "futures 0.3.32", + "grpcio", + "protobuf 2.8.0", + "protobuf-build", +] + +[[package]] +name = "tipb" +version = "0.0.1" +source = "git+https://github.com/pingcap/tipb.git#a4d204a193b4f9aa776343ac75281cf8442343ba" dependencies = [ "futures 0.3.32", "grpcio", @@ -7406,7 +7345,7 @@ source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud dependencies = [ "collections", "crossbeam-utils", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "lazy_static", "parking_lot 0.12.5", "pin-project", @@ -7434,7 +7373,7 @@ dependencies = [ "collections", "error_code", "farmhash", - "kvproto", + "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", "log_wrappers", "slog", "thiserror 1.0.69", @@ -7848,7 +7787,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/contrib/tiflash-columnar-hub/Cargo.toml b/contrib/tiflash-columnar-hub/Cargo.toml index a39f2b03f17..0099330ef4a 100644 --- a/contrib/tiflash-columnar-hub/Cargo.toml +++ b/contrib/tiflash-columnar-hub/Cargo.toml @@ -23,13 +23,14 @@ sysinfo = { git = "https://github.com/tikv/sysinfo", branch = "0.26-fix-cpu" } tokio-executor = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" } lindera = { git = "https://github.com/breezewish/lindera", branch = "v0.43.1-tokio-1.24" } tantivy = { git = "https://github.com/breezewish/tikv-tantivy.git", branch = "patch-0.22.1" } -pprof = { git = "https://github.com/tikv/pprof-rs.git", rev = "01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8" } +kvproto = { git = "https://github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } +tipb = { git = "https://github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } -[patch."https://github.com/pingcap/kvproto.git"] -kvproto = { git = "ssh://git@github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } +# [patch."https://github.com/pingcap/kvproto.git"] +# kvproto = { git = "ssh://git@github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } -[patch."https://github.com/pingcap/tipb.git"] -tipb = { git = "ssh://git@github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } +# [patch."https://github.com/pingcap/tipb.git"] +# tipb = { git = "ssh://git@github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } [patch."https://github.com/tidbcloud/cloud-storage-engine.git"] #api_version = { path = "../../../cloud-storage-engine/components/api_version" } diff --git a/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml b/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml index 4ab6344f085..ed22c90e614 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml +++ b/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml @@ -29,11 +29,11 @@ hyper = { version = "0.14.26", features = ["full", "runtime"] } keys = { workspace = true } kvengine = { workspace = true } kvenginepb = { workspace = true } -kvproto = { git = "ssh://git@github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } +kvproto = { git = "https://github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } lazy_static = "1.5.0" num_cpus = "1" pd_client = { workspace = true } -pprof = { version = "0.15", default-features = false, features = ["flamegraph", "cpp", "framehop-unwinder", "protobuf-codec"] } +pprof = { version = "0.15", default-features = false, features = ["flamegraph", "frame-pointer", "protobuf-codec"] } pprof_util = { version = "0.8.2", features = ["flamegraph"] } prometheus = { version = "0.13", features = ["nightly", "push"], default-features = true } protobuf = { version = "2.8", features = ["bytes"], default-features = true } @@ -47,7 +47,7 @@ slog-global = { git = "https://github.com/breeswish/slog-global.git", rev = "d59 tempfile = "3" thiserror = "1.0.44" tikv_util = { workspace = true } -tipb = { git = "ssh://git@github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } +tipb = { git = "https://github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } tokio = { version = "=1.26.0", features = ["fs", "full", "macros", "net", "process", "rt", "rt-multi-thread", "sync", "time"], default-features = true } # Keep tokio-executor aligned with the tokio-timer hotfix used by tikv_util. tokio-executor = "=0.1.9" From 303cb3b99121a7f41d7df515f9a114a7e0b94c24 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Wed, 3 Jun 2026 16:38:56 +0800 Subject: [PATCH 2/8] build: resolve the issue of multiple versions of kvproto/tipb Signed-off-by: JaySon-Huang --- contrib/tiflash-columnar-hub/Cargo.lock | 84 +++++++------------ contrib/tiflash-columnar-hub/Cargo.toml | 10 +-- .../hub-runtime/Cargo.toml | 4 +- 3 files changed, 34 insertions(+), 64 deletions(-) diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 9810d9d97f0..157be424c54 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -173,7 +173,7 @@ dependencies = [ "bytes", "codec", "engine_traits", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "match-template", "thiserror 1.0.69", "tikv_alloc", @@ -382,7 +382,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "hyper-tls 0.5.0", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "md5", "prometheus", @@ -1120,7 +1120,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "kvengine", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "pd_client", "rand 0.8.6", "serde", @@ -1404,7 +1404,7 @@ dependencies = [ "stable_deref_trait", "tantivy", "tempfile", - "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", + "tipb", "workspace-hack", ] @@ -1417,7 +1417,7 @@ dependencies = [ "derive_more", "error_code", "futures-io", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "openssl", "prometheus", @@ -1443,7 +1443,7 @@ dependencies = [ "fxhash", "hmac 0.10.1", "kvenginepb", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log", "openssl", @@ -2103,7 +2103,7 @@ dependencies = [ "futures 0.3.32", "futures-util", "hex 0.4.3", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "online_config", "openssl", @@ -2131,7 +2131,7 @@ dependencies = [ "error_code", "fail 0.5.1", "file_system", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "log_wrappers", "protobuf 2.8.0", "quick_cache", @@ -2221,7 +2221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2230,7 +2230,7 @@ version = "0.0.1" source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "grpcio", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "raft", "serde", @@ -3324,7 +3324,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi 0.5.2", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3427,7 +3427,7 @@ version = "0.1.0" source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "byteorder", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "log_wrappers", "thiserror 1.0.69", "tikv_alloc", @@ -3488,7 +3488,7 @@ dependencies = [ "itertools 0.10.5", "keys", "kvenginepb", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "libc", "log_wrappers", @@ -3531,7 +3531,7 @@ dependencies = [ "tikv_alloc", "tikv_util", "time 0.1.45", - "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", + "tipb", "tokio", "tokio-util", "tracker", @@ -3553,19 +3553,6 @@ dependencies = [ "workspace-hack", ] -[[package]] -name = "kvproto" -version = "0.0.2" -source = "git+https://github.com/pingcap/kvproto.git?rev=02dc8a9dc01827367f9d91ff2d88c447d3a838ab#02dc8a9dc01827367f9d91ff2d88c447d3a838ab" -dependencies = [ - "bytes", - "futures 0.3.32", - "grpcio", - "protobuf 2.8.0", - "protobuf-build", - "raft-proto", -] - [[package]] name = "kvproto" version = "0.0.2" @@ -4551,7 +4538,7 @@ dependencies = [ "grpcio", "hex 0.4.3", "http 0.2.12", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log", "log_wrappers", @@ -5734,7 +5721,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5747,7 +5734,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5863,7 +5850,7 @@ dependencies = [ "slog-term", "tidb_query_datatype", "tikv_util", - "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", + "tipb", "txn_types", "workspace-hack", ] @@ -5908,7 +5895,7 @@ dependencies = [ "http 0.2.12", "hyper 0.14.26", "hyper-rustls", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "prometheus", "rustls", @@ -5968,7 +5955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6665,7 +6652,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6778,7 +6765,7 @@ dependencies = [ "derive_more", "error_code", "futures 0.3.32", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log_wrappers", "maybe-async", @@ -6814,7 +6801,7 @@ dependencies = [ "encoding_rs 0.8.29", "error_code", "hex 0.4.3", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "log_wrappers", "match-template", @@ -6835,7 +6822,7 @@ dependencies = [ "tidb_query_common", "tikv_alloc", "tikv_util", - "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git)", + "tipb", "workspace-hack", ] @@ -6858,7 +6845,7 @@ dependencies = [ "keys", "kvengine", "kvenginepb", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git?rev=02dc8a9dc01827367f9d91ff2d88c447d3a838ab)", + "kvproto", "lazy_static", "num_cpus", "pd_client", @@ -6876,7 +6863,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tikv_util", - "tipb 0.0.1 (git+https://github.com/pingcap/tipb.git?rev=9fd33ded6f2c94633e601104cde26f2bdb5147d5)", + "tipb", "tokio", "tokio-executor", "toml", @@ -6946,7 +6933,7 @@ dependencies = [ "grpcio", "http 0.2.12", "hyper 0.14.26", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "libc", "log", @@ -7047,17 +7034,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tipb" -version = "0.0.1" -source = "git+https://github.com/pingcap/tipb.git?rev=9fd33ded6f2c94633e601104cde26f2bdb5147d5#9fd33ded6f2c94633e601104cde26f2bdb5147d5" -dependencies = [ - "futures 0.3.32", - "grpcio", - "protobuf 2.8.0", - "protobuf-build", -] - [[package]] name = "tipb" version = "0.0.1" @@ -7345,7 +7321,7 @@ source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud dependencies = [ "collections", "crossbeam-utils", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "lazy_static", "parking_lot 0.12.5", "pin-project", @@ -7373,7 +7349,7 @@ dependencies = [ "collections", "error_code", "farmhash", - "kvproto 0.0.2 (git+https://github.com/pingcap/kvproto.git)", + "kvproto", "log_wrappers", "slog", "thiserror 1.0.69", @@ -7787,7 +7763,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/contrib/tiflash-columnar-hub/Cargo.toml b/contrib/tiflash-columnar-hub/Cargo.toml index 0099330ef4a..44847971728 100644 --- a/contrib/tiflash-columnar-hub/Cargo.toml +++ b/contrib/tiflash-columnar-hub/Cargo.toml @@ -23,14 +23,8 @@ sysinfo = { git = "https://github.com/tikv/sysinfo", branch = "0.26-fix-cpu" } tokio-executor = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" } lindera = { git = "https://github.com/breezewish/lindera", branch = "v0.43.1-tokio-1.24" } tantivy = { git = "https://github.com/breezewish/tikv-tantivy.git", branch = "patch-0.22.1" } -kvproto = { git = "https://github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } -tipb = { git = "https://github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } - -# [patch."https://github.com/pingcap/kvproto.git"] -# kvproto = { git = "ssh://git@github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } - -# [patch."https://github.com/pingcap/tipb.git"] -# tipb = { git = "ssh://git@github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } +kvproto = { git = "https://github.com/pingcap/kvproto.git" } +tipb = { git = "https://github.com/pingcap/tipb.git" } [patch."https://github.com/tidbcloud/cloud-storage-engine.git"] #api_version = { path = "../../../cloud-storage-engine/components/api_version" } diff --git a/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml b/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml index ed22c90e614..e73251deda7 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml +++ b/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml @@ -29,7 +29,7 @@ hyper = { version = "0.14.26", features = ["full", "runtime"] } keys = { workspace = true } kvengine = { workspace = true } kvenginepb = { workspace = true } -kvproto = { git = "https://github.com/pingcap/kvproto.git", rev = "02dc8a9dc01827367f9d91ff2d88c447d3a838ab" } +kvproto = { git = "https://github.com/pingcap/kvproto.git" } lazy_static = "1.5.0" num_cpus = "1" pd_client = { workspace = true } @@ -47,7 +47,7 @@ slog-global = { git = "https://github.com/breeswish/slog-global.git", rev = "d59 tempfile = "3" thiserror = "1.0.44" tikv_util = { workspace = true } -tipb = { git = "https://github.com/pingcap/tipb.git", rev = "9fd33ded6f2c94633e601104cde26f2bdb5147d5" } +tipb = { git = "https://github.com/pingcap/tipb.git" } tokio = { version = "=1.26.0", features = ["fs", "full", "macros", "net", "process", "rt", "rt-multi-thread", "sync", "time"], default-features = true } # Keep tokio-executor aligned with the tokio-timer hotfix used by tikv_util. tokio-executor = "=0.1.9" From b85b654fbe4f0ca2d615d740e6a849f981f5a89d Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Wed, 3 Jun 2026 16:44:09 +0800 Subject: [PATCH 3/8] build: fix unused import Signed-off-by: JaySon-Huang --- contrib/tiflash-columnar-hub/hub-runtime/src/profile.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/tiflash-columnar-hub/hub-runtime/src/profile.rs b/contrib/tiflash-columnar-hub/hub-runtime/src/profile.rs index 82a3fff68fb..80087d444ec 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/src/profile.rs +++ b/contrib/tiflash-columnar-hub/hub-runtime/src/profile.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::pin::Pin; +#[cfg(feature = "external-jemalloc")] use std::{ ffi::{c_char, c_int, c_void, CString}, fs::File, io::BufReader, - pin::Pin, }; use futures::{ From d6df9b3bcb247ebd71a2c15ca23c52286f8af927 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 4 Jun 2026 11:11:01 +0800 Subject: [PATCH 4/8] build: remove git insteadOf Signed-off-by: JaySon-Huang --- release-linux-llvm/scripts/build-tiflash-release.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/release-linux-llvm/scripts/build-tiflash-release.sh b/release-linux-llvm/scripts/build-tiflash-release.sh index 65dd2b6d4e1..1ac1c46049e 100755 --- a/release-linux-llvm/scripts/build-tiflash-release.sh +++ b/release-linux-llvm/scripts/build-tiflash-release.sh @@ -117,7 +117,6 @@ if [ "${CMAKE_ENABLE_NEXT_GEN}" = "ON" ]; then # workaround compile issue of cloud-storage-engine dependency export CARGO_NET_GIT_FETCH_WITH_CLI=true - git config --global url."git@github.com:".insteadOf "https://github.com/" # prepare build dir and install dir for columnar if [ $CMAKE_BUILD_TYPE == "RELWITHDEBINFO" ]; then From 6256bea3b2997c6fc9e919ae3141988bfa8d75ad Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 4 Jun 2026 11:15:07 +0800 Subject: [PATCH 5/8] build: build tiflash-columnar first in release Signed-off-by: JaySon-Huang --- .../scripts/build-tiflash-release.sh | 102 +++++++++--------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/release-linux-llvm/scripts/build-tiflash-release.sh b/release-linux-llvm/scripts/build-tiflash-release.sh index 1ac1c46049e..2393f94dd70 100755 --- a/release-linux-llvm/scripts/build-tiflash-release.sh +++ b/release-linux-llvm/scripts/build-tiflash-release.sh @@ -61,54 +61,6 @@ NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} ENABLE_THINLTO=${ENABLE_THINLTO:-ON} ENABLE_PCH=${ENABLE_PCH:-ON} -INSTALL_DIR="${SRCPATH}/release-linux-llvm/tiflash" -rm -rf ${INSTALL_DIR} && mkdir -p ${INSTALL_DIR} - -if [ $CMAKE_BUILD_TYPE == "RELWITHDEBINFO" ]; then - BUILD_DIR="${SRCPATH}/release-linux-llvm/build-release" - ENABLE_FAILPOINTS="OFF" - JEMALLOC_NARENAS="-1" -else - BUILD_DIR="${SRCPATH}/release-linux-llvm/build-debug" - ENABLE_FAILPOINTS="ON" - JEMALLOC_NARENAS="40" -fi -rm -rf ${BUILD_DIR} && mkdir -p ${BUILD_DIR} -cd ${BUILD_DIR} - -cmake -S "${SRCPATH}" \ - ${DEFINE_CMAKE_PREFIX_PATH} \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DENABLE_TESTING=OFF \ - -DENABLE_TESTS=${ENABLE_TESTS} \ - -DENABLE_FAILPOINTS=${ENABLE_FAILPOINTS} \ - -DENABLE_NEXT_GEN=${CMAKE_ENABLE_NEXT_GEN} \ - -DJEMALLOC_NARENAS=${JEMALLOC_NARENAS} \ - -Wno-dev \ - -DUSE_CCACHE=OFF \ - -DUSE_INTERNAL_SSL_LIBRARY=ON \ - -DRUN_HAVE_STD_REGEX=0 \ - -DENABLE_THINLTO=${ENABLE_THINLTO} \ - -DTHINLTO_JOBS=${NPROC} \ - -DENABLE_PCH=${ENABLE_PCH} \ - -GNinja - -if [ $CMAKE_BUILD_TYPE == "TSAN" ] || [ $CMAKE_BUILD_TYPE == "ASAN" ]; then - cmake --build . --target gtests_dbms gtests_libcommon --parallel ${NPROC} - cmake --build . --target tiflash --parallel ${NPROC} - cmake --install . --component=tiflash-release --prefix="${INSTALL_DIR}" -else - cmake --build . --target tiflash --parallel ${NPROC} - cmake --install . --component=tiflash-release --prefix="${INSTALL_DIR}" - - # unset LD_LIBRARY_PATH before test - unset LD_LIBRARY_PATH - readelf -d "${INSTALL_DIR}/tiflash" - ldd "${INSTALL_DIR}/tiflash" - - # show version - ${INSTALL_DIR}/tiflash version -fi # If CMAKE_ENABLE_NEXT_GEN is enabled, build another binary with next-gen columnar features enabled and install to a different directory to avoid conflict with the non-columnar binary. if [ "${CMAKE_ENABLE_NEXT_GEN}" = "ON" ]; then @@ -160,3 +112,57 @@ if [ "${CMAKE_ENABLE_NEXT_GEN}" = "ON" ]; then # show version ${INSTALL_DIR}/tiflash version fi + + +if [ $CMAKE_BUILD_TYPE == "RELWITHDEBINFO" ]; then + ENABLE_FAILPOINTS="OFF" + JEMALLOC_NARENAS="-1" +else + ENABLE_FAILPOINTS="ON" + JEMALLOC_NARENAS="40" +fi + +INSTALL_DIR="${SRCPATH}/release-linux-llvm/tiflash" +rm -rf ${INSTALL_DIR} && mkdir -p ${INSTALL_DIR} +# prepare build_dir +if [ $CMAKE_BUILD_TYPE == "RELWITHDEBINFO" ]; then + BUILD_DIR="${SRCPATH}/release-linux-llvm/build-release" +else + BUILD_DIR="${SRCPATH}/release-linux-llvm/build-debug" +fi +rm -rf ${BUILD_DIR} && mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +cmake -S "${SRCPATH}" \ + ${DEFINE_CMAKE_PREFIX_PATH} \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DENABLE_TESTING=OFF \ + -DENABLE_TESTS=${ENABLE_TESTS} \ + -DENABLE_FAILPOINTS=${ENABLE_FAILPOINTS} \ + -DENABLE_NEXT_GEN=${CMAKE_ENABLE_NEXT_GEN} \ + -DJEMALLOC_NARENAS=${JEMALLOC_NARENAS} \ + -Wno-dev \ + -DUSE_CCACHE=OFF \ + -DUSE_INTERNAL_SSL_LIBRARY=ON \ + -DRUN_HAVE_STD_REGEX=0 \ + -DENABLE_THINLTO=${ENABLE_THINLTO} \ + -DTHINLTO_JOBS=${NPROC} \ + -DENABLE_PCH=${ENABLE_PCH} \ + -GNinja + +if [ $CMAKE_BUILD_TYPE == "TSAN" ] || [ $CMAKE_BUILD_TYPE == "ASAN" ]; then + cmake --build . --target gtests_dbms gtests_libcommon --parallel ${NPROC} + cmake --build . --target tiflash --parallel ${NPROC} + cmake --install . --component=tiflash-release --prefix="${INSTALL_DIR}" +else + cmake --build . --target tiflash --parallel ${NPROC} + cmake --install . --component=tiflash-release --prefix="${INSTALL_DIR}" + + # unset LD_LIBRARY_PATH before test + unset LD_LIBRARY_PATH + readelf -d "${INSTALL_DIR}/tiflash" + ldd "${INSTALL_DIR}/tiflash" + + # show version + ${INSTALL_DIR}/tiflash version +fi From c58a7e9529220ff39ed2258a102dffe1feeea715 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 4 Jun 2026 11:33:28 +0800 Subject: [PATCH 6/8] build: restore pprof changes Signed-off-by: JaySon-Huang --- contrib/tiflash-columnar-hub/Cargo.lock | 103 ++++++++++++++++-- contrib/tiflash-columnar-hub/Cargo.toml | 1 + .../hub-runtime/Cargo.toml | 2 +- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 157be424c54..237805a8250 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -14,7 +14,7 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ - "gimli", + "gimli 0.32.3", ] [[package]] @@ -952,7 +952,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.37.3", "rustc-demangle", "windows-link", ] @@ -1396,7 +1396,7 @@ dependencies = [ "charabia", "fastrand 2.4.1", "lazy_static", - "memmap2", + "memmap2 0.9.10", "once_cell", "ordered-float 5.3.0", "prost 0.13.5", @@ -1570,6 +1570,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpu-time" version = "1.0.0" @@ -2305,6 +2314,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "farmhash" version = "1.1.5" @@ -2447,6 +2462,20 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "framehop" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a5a3f0acb82df800ca3aa50c0d60d286c5d13d4cfc3114b3a9663f13b032fe" +dependencies = [ + "arrayvec", + "cfg-if", + "fallible-iterator 0.3.0", + "gimli 0.31.1", + "macho-unwind-info", + "pe-unwind-info", +] + [[package]] name = "fs2" version = "0.4.3" @@ -2691,6 +2720,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +dependencies = [ + "fallible-iterator 0.3.0", + "stable_deref_trait", +] + [[package]] name = "gimli" version = "0.32.3" @@ -2889,7 +2928,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f530c66081961311028f67d22da8e3f739dab6979fc200f199c3ef0fb2e194a" dependencies = [ - "fallible-iterator", + "fallible-iterator 0.2.0", ] [[package]] @@ -3494,7 +3533,7 @@ dependencies = [ "log_wrappers", "lz4", "maybe-async", - "memmap2", + "memmap2 0.9.10", "nix 0.24.3", "num_cpus", "online_config", @@ -3889,6 +3928,17 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +[[package]] +name = "macho-unwind-info" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149" +dependencies = [ + "thiserror 2.0.18", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "match-template" version = "0.0.1" @@ -3968,6 +4018,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memmap2" version = "0.9.10" @@ -4287,6 +4346,16 @@ dependencies = [ "url", ] +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "flate2", + "memchr", +] + [[package]] name = "object" version = "0.37.3" @@ -4563,6 +4632,19 @@ dependencies = [ "yatp", ] +[[package]] +name = "pe-unwind-info" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59" +dependencies = [ + "arrayvec", + "bitflags 2.11.1", + "thiserror 2.0.18", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -4680,17 +4762,19 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "pprof" version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" +source = "git+https://github.com/tikv/pprof-rs.git?rev=01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8#01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8" dependencies = [ "aligned-vec", "backtrace", "cfg-if", "findshlibs", + "framehop", "inferno 0.11.21", "libc", "log", + "memmap2 0.5.10", "nix 0.26.4", + "object 0.29.0", "once_cell", "protobuf 3.7.2", "protobuf-codegen 3.7.2", @@ -6414,7 +6498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" dependencies = [ "debugid", - "memmap2", + "memmap2 0.9.10", "stable_deref_trait", "uuid 1.23.1", ] @@ -6425,6 +6509,7 @@ version = "12.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" dependencies = [ + "cpp_demangle", "rustc-demangle", "symbolic-common", ] @@ -6522,7 +6607,7 @@ dependencies = [ "lru", "lz4_flex", "measure_time", - "memmap2", + "memmap2 0.9.10", "num_cpus", "once_cell", "oneshot", diff --git a/contrib/tiflash-columnar-hub/Cargo.toml b/contrib/tiflash-columnar-hub/Cargo.toml index 44847971728..9ec9ee7c6da 100644 --- a/contrib/tiflash-columnar-hub/Cargo.toml +++ b/contrib/tiflash-columnar-hub/Cargo.toml @@ -23,6 +23,7 @@ sysinfo = { git = "https://github.com/tikv/sysinfo", branch = "0.26-fix-cpu" } tokio-executor = { git = "https://github.com/tikv/tokio", branch = "tokio-timer-hotfix" } lindera = { git = "https://github.com/breezewish/lindera", branch = "v0.43.1-tokio-1.24" } tantivy = { git = "https://github.com/breezewish/tikv-tantivy.git", branch = "patch-0.22.1" } +pprof = { git = "https://github.com/tikv/pprof-rs.git", rev = "01cff82dbe6fe110a707bf2b38d8ebb1d14a18f8" } kvproto = { git = "https://github.com/pingcap/kvproto.git" } tipb = { git = "https://github.com/pingcap/tipb.git" } diff --git a/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml b/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml index e73251deda7..3a569c4777f 100644 --- a/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml +++ b/contrib/tiflash-columnar-hub/hub-runtime/Cargo.toml @@ -33,7 +33,7 @@ kvproto = { git = "https://github.com/pingcap/kvproto.git" } lazy_static = "1.5.0" num_cpus = "1" pd_client = { workspace = true } -pprof = { version = "0.15", default-features = false, features = ["flamegraph", "frame-pointer", "protobuf-codec"] } +pprof = { version = "0.15", default-features = false, features = ["flamegraph", "cpp", "framehop-unwinder", "protobuf-codec"] } pprof_util = { version = "0.8.2", features = ["flamegraph"] } prometheus = { version = "0.13", features = ["nightly", "push"], default-features = true } protobuf = { version = "2.8", features = ["bytes"], default-features = true } From 71f6d22982d801b7a5e206d5e2393f9266b0e7a3 Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 4 Jun 2026 11:34:37 +0800 Subject: [PATCH 7/8] build: fix unbound var Signed-off-by: JaySon-Huang --- .../scripts/build-tiflash-release.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/release-linux-llvm/scripts/build-tiflash-release.sh b/release-linux-llvm/scripts/build-tiflash-release.sh index 2393f94dd70..0e23095b225 100755 --- a/release-linux-llvm/scripts/build-tiflash-release.sh +++ b/release-linux-llvm/scripts/build-tiflash-release.sh @@ -60,7 +60,13 @@ SRCPATH=$( NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} ENABLE_THINLTO=${ENABLE_THINLTO:-ON} ENABLE_PCH=${ENABLE_PCH:-ON} - +if [ $CMAKE_BUILD_TYPE == "RELWITHDEBINFO" ]; then + ENABLE_FAILPOINTS="OFF" + JEMALLOC_NARENAS="-1" +else + ENABLE_FAILPOINTS="ON" + JEMALLOC_NARENAS="40" +fi # If CMAKE_ENABLE_NEXT_GEN is enabled, build another binary with next-gen columnar features enabled and install to a different directory to avoid conflict with the non-columnar binary. if [ "${CMAKE_ENABLE_NEXT_GEN}" = "ON" ]; then @@ -113,15 +119,6 @@ if [ "${CMAKE_ENABLE_NEXT_GEN}" = "ON" ]; then ${INSTALL_DIR}/tiflash version fi - -if [ $CMAKE_BUILD_TYPE == "RELWITHDEBINFO" ]; then - ENABLE_FAILPOINTS="OFF" - JEMALLOC_NARENAS="-1" -else - ENABLE_FAILPOINTS="ON" - JEMALLOC_NARENAS="40" -fi - INSTALL_DIR="${SRCPATH}/release-linux-llvm/tiflash" rm -rf ${INSTALL_DIR} && mkdir -p ${INSTALL_DIR} # prepare build_dir From 17576cb4e30e0cd213fb4a92bd770bbe825384ac Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Thu, 4 Jun 2026 13:47:13 +0800 Subject: [PATCH 8/8] Add cse as submodule Signed-off-by: JaySon-Huang --- .gitmodules | 6 + contrib/cloud-storage-engine | 1 + contrib/tiflash-columnar-hub/Cargo.lock | 286 ++++++++++++++++++++---- contrib/tiflash-columnar-hub/Cargo.toml | 20 +- 4 files changed, 264 insertions(+), 49 deletions(-) create mode 160000 contrib/cloud-storage-engine diff --git a/.gitmodules b/.gitmodules index e4cb41806e9..132b443afd6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -39,6 +39,12 @@ [submodule "contrib/tiflash-proxy-next-gen"] path = contrib/tiflash-proxy-next-gen url = git@github.com:tidbcloud/cloud-storage-engine.git +# cloud-storage-engine is a private repo. It is added as a submodule in TiFlash to bypass image +# release issues for tiflash-columnar-hub. This avoids private repo cloning in the internal CD pipeline. +[submodule "contrib/cloud-storage-engine"] + path = contrib/cloud-storage-engine + url = git@github.com:tidbcloud/cloud-storage-engine.git + branch = cloud-engine [submodule "contrib/prometheus-cpp"] path = contrib/prometheus-cpp url = https://github.com/jupp0r/prometheus-cpp.git diff --git a/contrib/cloud-storage-engine b/contrib/cloud-storage-engine new file mode 160000 index 00000000000..a9d93252f2a --- /dev/null +++ b/contrib/cloud-storage-engine @@ -0,0 +1 @@ +Subproject commit a9d93252f2ad0cba95eec51a857cd867cd5e6567 diff --git a/contrib/tiflash-columnar-hub/Cargo.lock b/contrib/tiflash-columnar-hub/Cargo.lock index 237805a8250..5efa22564b4 100644 --- a/contrib/tiflash-columnar-hub/Cargo.lock +++ b/contrib/tiflash-columnar-hub/Cargo.lock @@ -63,7 +63,6 @@ dependencies = [ [[package]] name = "aliyun" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "async-trait", "aws", @@ -167,7 +166,6 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "api_version" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "bitflags 1.3.2", "bytes", @@ -361,7 +359,6 @@ dependencies = [ [[package]] name = "aws" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "async-trait", "aws-config", @@ -1112,7 +1109,6 @@ dependencies = [ [[package]] name = "builtin_dfs" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "api_version", "async-trait", @@ -1183,7 +1179,6 @@ dependencies = [ [[package]] name = "case_macros" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "workspace-hack", ] @@ -1342,11 +1337,28 @@ dependencies = [ "atty", "bitflags 1.3.2", "strsim 0.8.0", - "textwrap", + "textwrap 0.11.0", "unicode-width", "vec_map", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_derive 3.2.25", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap 0.16.2", +] + [[package]] name = "clap" version = "4.6.1" @@ -1354,7 +1366,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", - "clap_derive", + "clap_derive 4.6.1", ] [[package]] @@ -1365,10 +1377,23 @@ checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", - "clap_lex", + "clap_lex 1.1.0", "strsim 0.11.1", ] +[[package]] +name = "clap_derive" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "clap_derive" version = "4.6.1" @@ -1381,6 +1406,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "1.1.0" @@ -1390,7 +1424,6 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clara_fts" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "anyhow", "charabia", @@ -1411,7 +1444,6 @@ dependencies = [ [[package]] name = "cloud" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "async-trait", "derive_more", @@ -1431,7 +1463,6 @@ dependencies = [ [[package]] name = "cloud_encryption" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "aliyun", "aws", @@ -1476,7 +1507,6 @@ dependencies = [ [[package]] name = "codec" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "byteorder", "error_code", @@ -1501,7 +1531,6 @@ dependencies = [ [[package]] name = "collections" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "fxhash", "tikv_alloc", @@ -2097,7 +2126,6 @@ dependencies = [ [[package]] name = "encryption" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "async-trait", "byteorder", @@ -2133,7 +2161,6 @@ dependencies = [ [[package]] name = "engine_traits" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "bytes", "case_macros", @@ -2236,7 +2263,6 @@ dependencies = [ [[package]] name = "error_code" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "grpcio", "kvproto", @@ -2350,7 +2376,6 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "file_system" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "collections", "crc32fast", @@ -2417,6 +2442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", + "libz-sys", "miniz_oxide", ] @@ -2879,6 +2905,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.5.0" @@ -3463,7 +3498,6 @@ dependencies = [ [[package]] name = "keys" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "byteorder", "kvproto", @@ -3477,7 +3511,6 @@ dependencies = [ [[package]] name = "kvengine" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "aligned-vec", "aliyun", @@ -3585,7 +3618,6 @@ dependencies = [ [[package]] name = "kvenginepb" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "protobuf 2.8.0", "protobuf-codegen-pure", @@ -3610,6 +3642,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.8", +] [[package]] name = "lazycell" @@ -3884,7 +3919,6 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "log_wrappers" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "hex 0.4.3", "protobuf 2.8.0", @@ -4088,6 +4122,15 @@ dependencies = [ "libc", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +dependencies = [ + "serde", +] + [[package]] name = "multimap" version = "0.10.1" @@ -4386,7 +4429,6 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "online_config" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "online_config_derive", "serde", @@ -4396,7 +4438,6 @@ dependencies = [ [[package]] name = "online_config_derive" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "proc-macro2", "quote", @@ -4487,6 +4528,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "outref" version = "0.5.2" @@ -4593,7 +4640,6 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pd_client" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "async-trait", "bstr", @@ -4779,7 +4825,7 @@ dependencies = [ "protobuf 3.7.2", "protobuf-codegen 3.7.2", "smallvec", - "spin", + "spin 0.10.0", "symbolic-demangle", "tempfile", "thiserror 2.0.18", @@ -4937,6 +4983,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", +] + [[package]] name = "prost" version = "0.12.6" @@ -4976,7 +5032,7 @@ dependencies = [ "heck 0.5.0", "itertools 0.14.0", "log", - "multimap", + "multimap 0.10.1", "once_cell", "petgraph", "prettyplease", @@ -4987,6 +5043,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-derive" version = "0.12.6" @@ -5414,7 +5483,6 @@ dependencies = [ [[package]] name = "recovery" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "dashmap 6.2.1", "http 0.2.12", @@ -5467,7 +5535,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata 0.4.14", - "regex-syntax", + "regex-syntax 0.8.10", ] [[package]] @@ -5475,6 +5543,9 @@ name = "regex-automata" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] [[package]] name = "regex-automata" @@ -5484,7 +5555,7 @@ checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.10", ] [[package]] @@ -5493,6 +5564,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -5915,7 +5992,6 @@ dependencies = [ [[package]] name = "schema" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "api_version", "async-trait", @@ -5964,7 +6040,6 @@ dependencies = [ [[package]] name = "security" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "aliyun", "aws", @@ -6404,6 +6479,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "spin" version = "0.10.0" @@ -6437,6 +6518,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -6675,7 +6762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" dependencies = [ "byteorder", - "regex-syntax", + "regex-syntax 0.8.10", "utf8-ranges", ] @@ -6743,7 +6830,6 @@ dependencies = [ [[package]] name = "tencentcloud" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "async-trait", "aws", @@ -6790,6 +6876,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + [[package]] name = "thiserror" version = "1.0.69" @@ -6842,7 +6934,6 @@ dependencies = [ [[package]] name = "tidb_query_common" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "anyhow", "api_version", @@ -6870,7 +6961,6 @@ dependencies = [ [[package]] name = "tidb_query_datatype" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "api_version", "base64 0.13.1", @@ -6983,10 +7073,40 @@ dependencies = [ "tonic", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tikv_alloc" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "fxhash", "lazy_static", @@ -6997,7 +7117,6 @@ dependencies = [ [[package]] name = "tikv_util" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "async-speed-limit", "backtrace", @@ -7233,6 +7352,7 @@ dependencies = [ "futures-io", "futures-sink", "pin-project-lite", + "slab", "tokio", "tracing", ] @@ -7312,7 +7432,6 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "trace_event" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "arc-swap 1.9.1", "bitflags 1.3.2", @@ -7331,6 +7450,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -7402,7 +7522,6 @@ dependencies = [ [[package]] name = "tracker" version = "0.0.1" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "collections", "crossbeam-utils", @@ -7426,7 +7545,6 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "txn_types" version = "0.1.0" -source = "git+https://github.com/tidbcloud/cloud-storage-engine.git?branch=cloud-engine#a9d93252f2ad0cba95eec51a857cd867cd5e6567" dependencies = [ "bitflags 1.3.2", "byteorder", @@ -8256,6 +8374,90 @@ dependencies = [ [[package]] name = "workspace-hack" version = "0.1.0" +dependencies = [ + "ahash", + "aho-corasick", + "base64 0.21.7", + "bytes", + "cc", + "chrono", + "clap 2.34.0", + "clap 3.2.25", + "crossbeam-queue", + "crossbeam-utils", + "data-encoding", + "deranged", + "digest 0.10.7", + "either", + "flate2", + "futures 0.3.32", + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", + "grpcio", + "grpcio-sys", + "hashbrown 0.14.5", + "hashbrown 0.15.5", + "heck 0.4.1", + "indexmap 1.9.3", + "indexmap 2.14.0", + "instant", + "itertools 0.10.5", + "lazy_static", + "libc", + "lindera-dictionary", + "log", + "memchr", + "multimap 0.8.3", + "nom 7.1.3", + "num 0.4.3", + "num-integer", + "num-iter", + "num-rational 0.4.2", + "num-traits", + "once_cell", + "openssl", + "phf_shared", + "proc-macro2", + "prost 0.11.9", + "protobuf 2.8.0", + "protobuf-build", + "rand 0.8.6", + "rand_core 0.6.4", + "regex", + "regex-automata 0.1.10", + "regex-automata 0.4.14", + "regex-syntax 0.8.10", + "reqwest 0.11.27", + "rustls", + "serde", + "serde_json", + "slog", + "smallvec", + "strum 0.27.2", + "syn 1.0.109", + "syn 2.0.117", + "tikv-jemalloc-ctl", + "tikv-jemalloc-sys", + "tikv-jemallocator", + "time 0.3.41", + "tokio", + "tokio-rustls", + "tokio-stream", + "tokio-util", + "tower", + "tracing", + "tracing-core", + "unicode-bidi", + "unicode-normalization", + "url", + "uuid 1.23.1", + "zstd-sys", +] [[package]] name = "xattr" @@ -8378,3 +8580,7 @@ dependencies = [ "cc", "pkg-config", ] + +[[patch.unused]] +name = "workspace-hack" +version = "0.1.0" diff --git a/contrib/tiflash-columnar-hub/Cargo.toml b/contrib/tiflash-columnar-hub/Cargo.toml index 9ec9ee7c6da..c3a3f52b48e 100644 --- a/contrib/tiflash-columnar-hub/Cargo.toml +++ b/contrib/tiflash-columnar-hub/Cargo.toml @@ -27,16 +27,18 @@ pprof = { git = "https://github.com/tikv/pprof-rs.git", rev = "01cff82dbe6fe110a kvproto = { git = "https://github.com/pingcap/kvproto.git" } tipb = { git = "https://github.com/pingcap/tipb.git" } +# cloud-storage-engine now is private. We add it as a submodule to workaround +# image release issue for building tiflash-columnar-hub in internal CD pipeline. [patch."https://github.com/tidbcloud/cloud-storage-engine.git"] -#api_version = { path = "../../../cloud-storage-engine/components/api_version" } -#builtin_dfs = { path = "../../../cloud-storage-engine/components/builtin_dfs" } -#cloud_encryption = { path = "../../../cloud-storage-engine/components/cloud_encryption" } -#keys = { path = "../../../cloud-storage-engine/components/keys" } -#kvengine = { path = "../../../cloud-storage-engine/components/kvengine" } -#kvenginepb = { path = "../../../cloud-storage-engine/components/kvenginepb" } -#pd_client = { path = "../../../cloud-storage-engine/components/pd_client" } -#security = { path = "../../../cloud-storage-engine/components/security" } -#tikv_util = { path = "../../../cloud-storage-engine/components/tikv_util" } +api_version = { path = "../cloud-storage-engine/components/api_version" } +builtin_dfs = { path = "../cloud-storage-engine/components/builtin_dfs" } +cloud_encryption = { path = "../cloud-storage-engine/components/cloud_encryption" } +keys = { path = "../cloud-storage-engine/components/keys" } +kvengine = { path = "../cloud-storage-engine/components/kvengine" } +kvenginepb = { path = "../cloud-storage-engine/components/kvenginepb" } +pd_client = { path = "../cloud-storage-engine/components/pd_client" } +security = { path = "../cloud-storage-engine/components/security" } +tikv_util = { path = "../cloud-storage-engine/components/tikv_util" } # cloud-storage-engine's generated workspace-hack currently forces # tikv-jemalloc-* into every consumer. That is fine for standalone