From 7247c5ac3a3744bbec9f001d4a367d2a61b79af1 Mon Sep 17 00:00:00 2001 From: Melody Ren Date: Fri, 31 Jul 2026 10:46:38 -0700 Subject: [PATCH 1/6] Make HSB transport configuration YAML-only Signed-off-by: Melody Ren --- .../cudaq/qec/realtime/decoding_config.h | 13 ++ .../decoding-server-cqr/CMakeLists.txt | 11 +- .../decoding-server-cqr/DecodingServer.cpp | 15 +- .../decoding-server-cqr/DecodingServer.h | 7 +- .../DeviceGraphFactory.cpp | 14 +- .../DeviceGraphLinkCheck.cpp | 17 +- .../DeviceGraphTransceiver.cpp | 150 ++++-------------- .../DeviceGraphTransceiver.h | 44 ++--- .../tools/decoding-server/decoding_server.cpp | 71 +++------ .../realtime/test_decoding_server.cpp | 59 +++++++ libs/qec/unittests/test_decoders_yaml.cpp | 5 + .../utils/hsb_fpga_decoding_server_test.sh | 77 ++++++--- 12 files changed, 242 insertions(+), 241 deletions(-) diff --git a/libs/qec/include/cudaq/qec/realtime/decoding_config.h b/libs/qec/include/cudaq/qec/realtime/decoding_config.h index 868e2cf67..025c688ad 100644 --- a/libs/qec/include/cudaq/qec/realtime/decoding_config.h +++ b/libs/qec/include/cudaq/qec/realtime/decoding_config.h @@ -135,6 +135,19 @@ struct transport_config { std::vector args; transport_shape_override device_graph; + /// Resolve the provider and ordered arguments for a device-graph ring. + /// The shape-specific provider overrides the section provider, while its + /// arguments are appended to the section arguments. + transport_shape_override resolve_device_graph() const { + transport_shape_override resolved; + resolved.provider = + device_graph.provider.empty() ? provider : device_graph.provider; + resolved.args = args; + resolved.args.insert(resolved.args.end(), device_graph.args.begin(), + device_graph.args.end()); + return resolved; + } + bool operator==(const transport_config &) const = default; }; diff --git a/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt b/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt index cf3eeee60..086a8b6dc 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt +++ b/libs/qec/lib/realtime/decoding-server-cqr/CMakeLists.txt @@ -146,10 +146,9 @@ set_target_properties(cudaq-qec-decoding-server PROPERTIES # (DeviceGraphFactory.cpp) of the factory that DecodingServer.cpp references # weakly. Kept out of the core library so core consumers stay loadable on # machines without the CUDA runtime. The transport itself is a runtime- -# loaded bridge provider (libcudaq-realtime-bridge-hololink.so or a -# CUDAQ_REALTIME_BRIDGE_LIB drop-in), so this component links only the -# CUDA-Q realtime bridge C API and the CUDA runtime -- no Hololink / DOCA / -# HSB link-time dependencies. +# loaded bridge provider selected by the YAML transport section, so this +# component links only the CUDA-Q realtime bridge C API and the CUDA runtime -- +# no Hololink / DOCA / HSB link-time dependencies. # # Consumers must link this WHOLE_ARCHIVE: the only reference to the factory # is weak, which does not pull archive members on its own. @@ -162,7 +161,9 @@ if(CUDAQ_QEC_DEVICE_GRAPH_AVAILABLE) ) target_compile_definitions(cudaq-qec-decoding-server-device-graph - PRIVATE CUDAQ_QEC_DEVICE_GRAPH_AVAILABLE) + PRIVATE + CUDAQ_QEC_DEVICE_GRAPH_AVAILABLE + QEC_BRIDGE_PROVIDER_DIR="${_cudaq_rt_lib_dir}") target_include_directories(cudaq-qec-decoding-server-device-graph PRIVATE "${CUDAQ_REALTIME_INCLUDE_DIR}" diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp index 6bef45d3b..bba565aca 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.cpp @@ -30,11 +30,14 @@ // definition of this factory; anywhere else the weak reference is null and // make_transport throws. extern "C" __attribute__((weak)) cudaq::qec::decoding_server::ITransceiver * -cudaqx_qec_make_device_graph_transceiver(int pinned_cuda_device); +cudaqx_qec_make_device_graph_transceiver( + int pinned_cuda_device, + const cudaq::qec::decoding::config::transport_shape_override *transport); namespace cudaq::qec::decoding_server { using cudaq::qec::decoding::config::DecoderDispatch; +using cudaq::qec::decoding::config::transport_shape_override; using cudaq::qec::decoding::rpc::EnqueueRequestPayload; using cudaq::qec::decoding::rpc::GetCorrectionsRequestPayload; using cudaq::qec::decoding::rpc::kEnqueueSyndromesFunctionId; @@ -58,8 +61,8 @@ int resolve_decode_device(int decoder_pin) { } std::unique_ptr -DecodingServer::make_transport(DecoderDispatch dispatch, - int pinned_cuda_device) { +DecodingServer::make_transport(DecoderDispatch dispatch, int pinned_cuda_device, + const transport_shape_override &transport) { switch (dispatch) { case DecoderDispatch::device_graph: // device_graph lives in the cudaq-qec-decoding-server-device-graph @@ -68,7 +71,8 @@ DecodingServer::make_transport(DecoderDispatch dispatch, // transceiver config lives; we just thread the pin to it. if (cudaqx_qec_make_device_graph_transceiver) return std::unique_ptr( - cudaqx_qec_make_device_graph_transceiver(pinned_cuda_device)); + cudaqx_qec_make_device_graph_transceiver(pinned_cuda_device, + &transport)); throw std::runtime_error( "device_graph dispatch requested but the device-graph component is " "not linked into this binary. Link " @@ -112,7 +116,8 @@ DecodingServer::DecodingServer(const std::string &config_yaml) { boot_sessions.size() == 1 ? boot_sessions.begin()->second->dec->get_cuda_device_id() : -1; - auto t = make_transport(dispatch, pinned_cuda_device); + auto t = make_transport(dispatch, pinned_cuda_device, + config.transport.resolve_device_graph()); ITransceiver *raw = t.get(); owned_transports_.push_back(std::move(t)); function_transport_[kEnqueueSyndromesFunctionId] = raw; diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h index d47bf08c0..e561b0d97 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h +++ b/libs/qec/lib/realtime/decoding-server-cqr/DecodingServer.h @@ -101,9 +101,10 @@ class DecodingServer { /// Create a transceiver for \p transport_type. Throws for RoCE transports /// until per-session transceiver adapters are /// available via CUDAQ_REALTIME. - static std::unique_ptr - make_transport(cudaq::qec::decoding::config::DecoderDispatch dispatch, - int pinned_cuda_device); + static std::unique_ptr make_transport( + cudaq::qec::decoding::config::DecoderDispatch dispatch, + int pinned_cuda_device, + const cudaq::qec::decoding::config::transport_shape_override &transport); // Destruction order matters: the device-graph scheduler (inside // owned_transports_) holds a cudaGraphExec_t captured from a session's diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphFactory.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphFactory.cpp index da3112dae..53dc4e392 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphFactory.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphFactory.cpp @@ -18,12 +18,22 @@ #include "DecodingServer.h" // resolve_decode_device (core symbol) #include "DeviceGraphTransceiver.h" +#include + extern "C" cudaq::qec::decoding_server::ITransceiver * -cudaqx_qec_make_device_graph_transceiver(int pinned_cuda_device) { +cudaqx_qec_make_device_graph_transceiver( + int pinned_cuda_device, + const cudaq::qec::decoding::config::transport_shape_override *transport) { using namespace cudaq::qec::decoding_server; + if (!transport) + throw std::invalid_argument( + "device-graph transport configuration is missing"); + // The device-graph GPU is the decoder's cuda_device_id pin; resolve it // here, inside the component, where DeviceGraphConfig is visible. - auto cfg = DeviceGraphConfig::from_env(); + DeviceGraphConfig cfg; + cfg.provider = transport->provider; + cfg.provider_args = transport->args; cfg.gpu_id = resolve_decode_device(pinned_cuda_device); return new DeviceGraphTransceiver(cfg); } diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphLinkCheck.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphLinkCheck.cpp index b7495c127..9343cd50e 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphLinkCheck.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphLinkCheck.cpp @@ -7,9 +7,9 @@ ******************************************************************************/ // Link canary for the device-graph component -- not meant to be executed -// (running it would require QEC_DEVICE_GRAPH_* env, a GPU driver, and -// RDMA-capable -// hardware). Building it forces the linker to resolve DeviceGraphTransceiver's +// (running it would require a transport provider, a GPU driver, and +// RDMA-capable hardware). Building it forces the linker to resolve +// DeviceGraphTransceiver's // full dependency chain (hololink, DOCA, CUDA driver stubs), so HSB API // drift is caught at build time even on machines where nothing links the // component into a runnable binary (driverless CI: the decoding_server @@ -19,12 +19,17 @@ namespace cudaq::qec::decoding_server { struct ITransceiver; } +namespace cudaq::qec::decoding::config { +struct transport_shape_override; +} extern "C" cudaq::qec::decoding_server::ITransceiver * -cudaqx_qec_make_device_graph_transceiver(int pinned_cuda_device); +cudaqx_qec_make_device_graph_transceiver( + int pinned_cuda_device, + const cudaq::qec::decoding::config::transport_shape_override *transport); -using DeviceGraphFactoryFn = - cudaq::qec::decoding_server::ITransceiver *(*)(int); +using DeviceGraphFactoryFn = cudaq::qec::decoding_server::ITransceiver + *(*)(int, const cudaq::qec::decoding::config::transport_shape_override *); static DeviceGraphFactoryFn volatile device_graph_factory = &cudaqx_qec_make_device_graph_transceiver; diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp index 911144d5c..2ce98580e 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp @@ -10,7 +10,6 @@ #include "DeviceGraphTransceiver.h" #include "cudaq/qec/logger.h" -#include "cudaq/qec/realtime/decoder_rpc_wire_format.h" #include "cudaq/qec/realtime/graph_resources.h" #include @@ -18,71 +17,34 @@ #include #include #include +#include #include -#include -#include #include #include #include -#include #include -// CUDAQ device-graph scheduler types (cudaq-realtime-dispatch) and the -// RPCHeader wire struct the provider's --payload-size argument is defined -// against. +// CUDA-Q realtime bridge-provider interface. #include "cudaq/realtime/hololink_bridge_common.h" namespace cudaq::qec::decoding_server { -// --------------------------------------------------------------------------- -// Internal helpers (same pattern as hololink_qldpc_graph_decoder_bridge.cpp) -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// DeviceGraphConfig::from_env -// --------------------------------------------------------------------------- - -// Each knob reads QEC_DEVICE_GRAPH_; the values are forwarded to -// whatever transport provider is loaded. -static const char *env_raw(const char *name) { - const std::string full = std::string("QEC_DEVICE_GRAPH_") + name; - return std::getenv(full.c_str()); -} -static std::string env_str(const char *name, const char *def = "") { - const char *v = env_raw(name); - return v ? v : def; -} -static uint32_t env_u32(const char *name, uint32_t def) { - const char *v = env_raw(name); - return v ? static_cast(std::stoul(v)) : def; -} -static size_t env_size(const char *name, size_t def) { - const char *v = env_raw(name); - return v ? static_cast(std::stoull(v)) : def; +namespace { + +std::string resolve_provider_library(const std::string &provider) { + if (provider.find('/') != std::string::npos) + return provider; + const std::string soname = "libcudaq-realtime-bridge-" + provider + ".so"; +#ifdef QEC_BRIDGE_PROVIDER_DIR + const std::string candidate = + std::string(QEC_BRIDGE_PROVIDER_DIR) + "/" + soname; + if (std::ifstream(candidate).good()) + return candidate; +#endif + return soname; } -DeviceGraphConfig DeviceGraphConfig::from_env() { - DeviceGraphConfig c; - c.device_name = env_str("DEVICE"); - c.peer_ip = env_str("PEER_IP"); - c.remote_qp = env_u32("REMOTE_QP", 0); - // gpu_id is not read from the environment: the device is the decoder's - // cuda_device_id, resolved by resolve_decode_device() at transport - // creation. - c.frame_size = env_size("FRAME_SIZE", 0); - c.page_size = env_size("PAGE_SIZE", 0); // 0 → derived from frame_size - c.num_pages = env_size("NUM_PAGES", 0); - // Generic pass-through channel for providers whose argument surface does - // not match the named knobs above: whitespace-separated tokens, forwarded - // to the provider verbatim (after the named arguments). - if (const char *extra = env_raw("PROVIDER_ARGS")) { - std::istringstream in(extra); - std::string token; - while (in >> token) - c.extra_args.push_back(token); - } - return c; -} +} // namespace // --------------------------------------------------------------------------- // DeviceGraphTransceiver constructor @@ -90,48 +52,10 @@ DeviceGraphConfig DeviceGraphConfig::from_env() { DeviceGraphTransceiver::DeviceGraphTransceiver(const DeviceGraphConfig &config) : gpu_id_(config.gpu_id) { - if (config.device_name.empty()) - throw std::runtime_error( - "DeviceGraphTransceiver: QEC_DEVICE_GRAPH_DEVICE not set"); - if (config.peer_ip.empty()) + if (config.provider.empty()) throw std::runtime_error( - "DeviceGraphTransceiver: QEC_DEVICE_GRAPH_PEER_IP not set"); - if (config.remote_qp == 0) - throw std::runtime_error( - "DeviceGraphTransceiver: QEC_DEVICE_GRAPH_REMOTE_QP not set"); - - // Derive page_size from frame_size if not overridden, then round up to the - // 128-byte Hololink granularity. Mirrors the derivation in - // hololink_qldpc_graph_decoder_bridge.cpp (lines 279-282). - size_t page_size = config.page_size ? config.page_size : config.frame_size; - page_size = (page_size + 127) & ~static_cast(127); - - if (page_size != 0 && - config.num_pages > std::numeric_limits::max() / page_size) - throw std::runtime_error( - "DeviceGraphTransceiver: ring size overflow for " - "QEC_DEVICE_GRAPH_FRAME_SIZE/QEC_DEVICE_GRAPH_PAGE_SIZE=" + - std::to_string(page_size) + - " and QEC_DEVICE_GRAPH_NUM_PAGES=" + std::to_string(config.num_pages)); - const size_t ring_bytes = page_size * config.num_pages; - const long host_page_size = ::sysconf(_SC_PAGESIZE); - if (host_page_size > 0 && - ring_bytes % static_cast(host_page_size) != 0) - throw std::runtime_error("DeviceGraphTransceiver: ring buffer size " + - std::to_string(ring_bytes) + - " bytes is not aligned to host page size " + - std::to_string(host_page_size) + - " bytes; adjust QEC_DEVICE_GRAPH_NUM_PAGES or " - "QEC_DEVICE_GRAPH_PAGE_SIZE"); - - // The provider computes frame_size = sizeof(RPCHeader) + payload_size, so - // hand it the payload remainder of our frame budget. - if (config.frame_size < sizeof(cudaq::realtime::RPCHeader)) - throw std::runtime_error( - "DeviceGraphTransceiver: QEC_DEVICE_GRAPH_FRAME_SIZE smaller than " - "the RPC header"); - const size_t payload_size = - config.frame_size - sizeof(cudaq::realtime::RPCHeader); + "DeviceGraphTransceiver: device_graph transport provider must be set " + "in YAML"); // Bring the Hololink transceiver up through the bridge-provider interface: // create() = hololink_create_transceiver + hololink_start (3-kernel shape: @@ -141,39 +65,25 @@ DeviceGraphTransceiver::DeviceGraphTransceiver(const DeviceGraphConfig &config) // follows the C argv convention and starts parsing at argv[1] -- without // the placeholder the first real option would be silently skipped (and the // bridge would fall back to its built-in device default). - const std::vector args = { - "device-graph-transceiver", - "--device=" + config.device_name, - "--peer-ip=" + config.peer_ip, - "--remote-qp=" + std::to_string(config.remote_qp), - "--gpu=" + std::to_string(config.gpu_id), - "--page-size=" + std::to_string(page_size), - "--num-pages=" + std::to_string(config.num_pages), - "--payload-size=" + std::to_string(payload_size), - }; + std::vector args{"device-graph-transceiver"}; + args.insert(args.end(), config.provider_args.begin(), + config.provider_args.end()); + // The decoder's YAML cuda_device_id is authoritative for graph capture, + // provider rings, and scheduler launch, so place it last. + args.push_back("--gpu=" + std::to_string(config.gpu_id)); std::vector argv; argv.reserve(args.size()); for (auto &a : args) - argv.push_back(const_cast(a.c_str())); - - // A provider is just a library name/path to the loader (cached per - // process, keyed by that string). Default to the hololink GPU-RoCE - // provider shipped with cudaq-realtime; CUDAQ_REALTIME_BRIDGE_LIB names a - // replacement library (same mechanism as the decoding server's - // --transport=.so partner drop-in). - const char *env_lib = std::getenv("CUDAQ_REALTIME_BRIDGE_LIB"); - const std::string provider_lib = - env_lib ? env_lib : "libcudaq-realtime-bridge-hololink.so"; + argv.push_back(a.data()); + + const std::string provider_lib = resolve_provider_library(config.provider); if (cudaq_bridge_create_from_library(&bridge_, provider_lib.c_str(), static_cast(argv.size()), argv.data()) != CUDAQ_OK || !bridge_) throw std::runtime_error( - "DeviceGraphTransceiver: bridge provider create failed for device=" + - config.device_name + " peer=" + config.peer_ip + " (is " + - provider_lib + - " on the loader path, and " - "does the IB netdev have an IPv4 address assigned for RoCE v2 GID?)"); + "DeviceGraphTransceiver: bridge provider create failed for '" + + config.provider + "' (resolved as " + provider_lib + ")"); // Adopt the DOCA ring buffer GPU VRAM pointers from the provider. cudaq_ringbuffer_t ring{}; diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.h b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.h index 8ff8028c6..49b206b89 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.h +++ b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.h @@ -34,36 +34,15 @@ struct cudaq_dispatch_graph_context; namespace cudaq::qec::decoding_server { -/// Runtime configuration for DeviceGraphTransceiver. Transport fields are -/// read from environment variables so that the server can be reconfigured -/// without a rebuild; gpu_id is the exception -- it is the decoder's -/// cuda_device_id, filled in at transport creation. -/// -/// The named fields are OPTIONAL convenience knobs: each becomes a -/// `--=` argument to the provider only when its environment variable -/// is set (they match the built-in hololink provider's flags; an RDMA-style -/// provider will typically want the same shape). A provider with a -/// different argument surface is configured through `extra_args` -/// (QEC_DEVICE_GRAPH_PROVIDER_ARGS, whitespace-separated tokens forwarded -/// verbatim -- the same pass-through contract as the decoding server's -/// provider args). Providers should ignore arguments they do not -/// recognize. +/// Runtime configuration for DeviceGraphTransceiver. Provider identity and +/// arguments come from the YAML transport section; gpu_id comes from the +/// decoder entry's cuda_device_id. struct DeviceGraphConfig { - std::string - device_name; ///< QEC_DEVICE_GRAPH_DEVICE (IB netdev, e.g. "mlx5_0") - uint32_t remote_qp{0}; ///< QEC_DEVICE_GRAPH_REMOTE_QP (FPGA/emulator QP) - int gpu_id{0}; ///< FPGA-affine GPU; set from the decoder's - ///< cuda_device_id by resolve_decode_device() - size_t frame_size{0}; ///< QEC_DEVICE_GRAPH_FRAME_SIZE (max RPC frame bytes) - size_t page_size{0}; ///< QEC_DEVICE_GRAPH_PAGE_SIZE (0 → from frame_size) - size_t num_pages{0}; ///< QEC_DEVICE_GRAPH_NUM_PAGES (ring depth) - std::string peer_ip; ///< QEC_DEVICE_GRAPH_PEER_IP (FPGA/emulator IPv4) - std::vector - extra_args; ///< QEC_DEVICE_GRAPH_PROVIDER_ARGS, forwarded verbatim + std::string provider; ///< YAML transport provider/path + std::vector provider_args; ///< YAML transport arguments + int gpu_id{0}; ///< FPGA-affine GPU, resolved from decoder cuda_device_id // (QEC_DEVICE_GRAPH_RESERVED_SMS is consumed by DecodingSession, where the // decode graph is captured.) - - static DeviceGraphConfig from_env(); }; /// Device-graph dispatch engine (transport-blind) for the decoding server. @@ -72,12 +51,11 @@ struct DeviceGraphConfig { /// /// The Hololink transceiver (DOCA GPU ring buffers fed by FPGA RDMA writes) /// is brought up through the CUDA-Q realtime bridge-provider interface: the -/// constructor loads the provider (the built-in -/// `libcudaq-realtime-bridge-hololink.so`, or the library named by -/// `CUDAQ_REALTIME_BRIDGE_LIB` when set), and adopts the provider's -/// RING_BUFFER context. `launch_scheduler()` wires those ring buffers to the -/// CUDAQ device-graph scheduler (`cudaq_create_dispatch_graph_regular`) and -/// the captured decoder CUDA graph, then starts the provider's I/O loop. +/// constructor loads the provider selected by the YAML transport section and +/// adopts its RING_BUFFER context. `launch_scheduler()` wires those ring +/// buffers to the CUDAQ device-graph scheduler +/// (`cudaq_create_dispatch_graph_regular`) and the captured decoder CUDA graph, +/// then starts the provider's I/O loop. /// /// Consequence of the provider split: this library needs only the CUDA-Q /// realtime headers + libcudaq-realtime.so at build time; the Hololink / diff --git a/libs/qec/tools/decoding-server/decoding_server.cpp b/libs/qec/tools/decoding-server/decoding_server.cpp index e27fb5992..3ef35a792 100644 --- a/libs/qec/tools/decoding-server/decoding_server.cpp +++ b/libs/qec/tools/decoding-server/decoding_server.cpp @@ -58,8 +58,7 @@ /// dispatcher below; a device_graph decoder routes the whole server through /// the CQR DecodingServer, whose DeviceGraphTransceiver runs the /// self-relaunching GPU scheduler over the same kind of runtime-loaded -/// provider (the hololink library by default; the YAML transport section or -/// the --transport fallback selects another). +/// provider selected and configured by the YAML transport section. #include "cudaq/qec/realtime/decoding_config.h" @@ -93,6 +92,7 @@ #include #include #include +#include #include extern "C" void cudaqx_qec_realtime_device_call_service_force_link(); @@ -348,45 +348,22 @@ int main(int argc, char **argv) { #ifdef QEC_HAVE_DEVICE_GRAPH_DISPATCH if (all_device_graph) { // DecodingServer(config_yaml) reads the YAML, creates the - // DeviceGraphTransceiver (which loads a bridge provider: the built-in - // hololink one, or CUDAQ_REALTIME_BRIDGE_LIB), loads decoder sessions, - // and calls launch_scheduler() to wire the CUDAQ device-graph scheduler - // to the provider's GPU rings. The GPU scheduler then handles + // DeviceGraphTransceiver from the YAML transport section, loads decoder + // sessions, and calls launch_scheduler() to wire the CUDA-Q device-graph + // scheduler to the provider's GPU rings. The GPU scheduler then handles // RX→dispatch→decode→TX autonomously; this thread just waits for signal. // // Construction throws when the device-graph component is not linked into // this binary (no proprietary cudevice archive) or when provider // bring-up fails. // - // Provider resolution for the standalone transceiver mirrors the - // per-ring loop below: the transport section's device_graph shape - // override > the section's provider > the --transport CLI fallback > - // the transceiver's built-in default (hololink). A YAML that names a - // provider plus a CLI --transport is rejected before reaching here. - std::string dg_provider; - if (!decoder_config.transport.device_graph.provider.empty()) - dg_provider = decoder_config.transport.device_graph.provider; - else if (!decoder_config.transport.provider.empty()) - dg_provider = decoder_config.transport.provider; - else if (cfg.transport_from_cli) - dg_provider = cfg.transport; - if (!dg_provider.empty()) - ::setenv("CUDAQ_REALTIME_BRIDGE_LIB", - resolve_provider_lib(dg_provider).c_str(), /*overwrite=*/1); - // Provider args from the transport section ride the same generic - // pass-through the per-ring loop uses: section args first, then the - // device_graph shape override's args. The transceiver forwards them - // verbatim after its named knobs (QEC_DEVICE_GRAPH_* env), so a - // non-HSB provider is configured entirely from the YAML. - { - std::string dg_args; - for (const auto &a : decoder_config.transport.args) - dg_args += (dg_args.empty() ? "" : " ") + a; - for (const auto &a : decoder_config.transport.device_graph.args) - dg_args += (dg_args.empty() ? "" : " ") + a; - if (!dg_args.empty()) - ::setenv("QEC_DEVICE_GRAPH_PROVIDER_ARGS", dg_args.c_str(), - /*overwrite=*/1); + // The standalone device-graph transport has no CLI or environment + // fallback: the deployment YAML is its single source of truth. + if (decoder_config.transport.resolve_device_graph().provider.empty()) { + std::cerr << "ERROR: an all-device_graph config must name its transport " + "provider in the YAML transport section" + << std::endl; + return 1; } try { cudaq::qec::decoding_server::DecodingServer server(cfg.config_path); @@ -485,6 +462,7 @@ int main(int argc, char **argv) { struct DecoderRing { std::int64_t decoder_id = 0; bool device_graph = false; + int gpu_id = 0; cudaq_realtime_bridge_handle_t bridge = nullptr; std::uint32_t num_slots = 0; std::uint32_t slot_size = 0; @@ -532,6 +510,8 @@ int main(int argc, char **argv) { const auto &dc = decoder_config.decoders[i]; ring.decoder_id = dc.id; ring.device_graph = (dc.dispatch == config::DecoderDispatch::device_graph); + if (ring.device_graph) + ring.gpu_id = dc.cuda_device_id.value_or(0); // The wire is deployment config, resolved from the YAML's top-level // `transport:` section (never from decoder entries). Per-ring @@ -544,14 +524,14 @@ int main(int argc, char **argv) { // different provider libraries in one process. const auto &transport_section = decoder_config.transport; std::string ring_provider_name = default_provider; - std::vector ring_extra_args = transport_section.args; + std::vector ring_extra_args; if (ring.device_graph) { - if (!transport_section.device_graph.provider.empty()) - ring_provider_name = transport_section.device_graph.provider; - ring_extra_args.insert(ring_extra_args.end(), - transport_section.device_graph.args.begin(), - transport_section.device_graph.args.end()); - } + const auto resolved = transport_section.resolve_device_graph(); + if (!resolved.provider.empty()) + ring_provider_name = resolved.provider; + ring_extra_args = std::move(resolved.args); + } else + ring_extra_args = transport_section.args; const std::string ring_lib = resolve_provider_lib(ring_provider_name); std::vector ring_argv = provider_argv; for (auto &a : ring_extra_args) @@ -671,12 +651,9 @@ int main(int argc, char **argv) { teardown_rings(); return 1; } - const int gpu_id = [] { - const char *value = std::getenv("QEC_DEVICE_GRAPH_GPU_ID"); - return value ? std::atoi(value) : 0; - }(); ring.dg_consumer = cudaqx_qec_make_device_graph_ring_consumer( - &ringbuffer, ring.num_slots, ring.slot_size, gpu_id, graph_resources); + &ringbuffer, ring.num_slots, ring.slot_size, ring.gpu_id, + graph_resources); if (!ring.dg_consumer) { std::cerr << "ERROR: device-graph scheduler launch failed (decoder " << ring.decoder_id << "; see log above)" << std::endl; diff --git a/libs/qec/unittests/realtime/test_decoding_server.cpp b/libs/qec/unittests/realtime/test_decoding_server.cpp index 2d21d1797..55c5bb1bf 100644 --- a/libs/qec/unittests/realtime/test_decoding_server.cpp +++ b/libs/qec/unittests/realtime/test_decoding_server.cpp @@ -44,8 +44,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -157,6 +159,26 @@ std::string env_or(const char *name, const std::string &fallback) { return value && *value ? std::string(value) : fallback; } +class ScopedEnv { +public: + ScopedEnv(const char *name, const char *value) : name_(name) { + if (const char *old_value = std::getenv(name)) + old_value_ = old_value; + ::setenv(name, value, 1); + } + + ~ScopedEnv() { + if (old_value_) + ::setenv(name_.c_str(), old_value_->c_str(), 1); + else + ::unsetenv(name_.c_str()); + } + +private: + std::string name_; + std::optional old_value_; +}; + // The server binary path is baked in at configure time (the server is built // from libs/qec/tools/decoding-server); QEC_DECODING_SERVER overrides it. The // example decoder configs are placed in the same directory as the server. @@ -624,6 +646,43 @@ TEST(DecodingServerTwoProcess, TransportCliConflictsWithYamlSection) { << server.captured; } +TEST(DecodingServerTwoProcess, DeviceGraphRequiresYamlTransportProvider) { + ScopedEnv legacy_provider("CUDAQ_REALTIME_BRIDGE_LIB", + "/tmp/legacy-provider.so"); + ScopedEnv legacy_device("QEC_DEVICE_GRAPH_DEVICE", "mlx5_legacy"); + ScopedEnv legacy_peer_ip("QEC_DEVICE_GRAPH_PEER_IP", "192.0.2.1"); + ScopedEnv legacy_remote_qp("QEC_DEVICE_GRAPH_REMOTE_QP", "17"); + ScopedEnv legacy_frame_size("QEC_DEVICE_GRAPH_FRAME_SIZE", "384"); + ScopedEnv legacy_num_pages("QEC_DEVICE_GRAPH_NUM_PAGES", "64"); + + const std::string config_path = + ::testing::TempDir() + "/decoding_server_device_graph_no_transport.yaml"; + { + std::ofstream config_file(config_path); + config_file << "decoders:\n" + << " - id: 0\n" + << " type: single_error_lut\n" + << " dispatch: device_graph\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n"; + } + + ServerProcess server; + std::string error; + EXPECT_FALSE(server.start(config_path, error, 8000, + /*transport_cli=*/false, + /*capture_stderr=*/true)) + << "server unexpectedly reached READY: " << server.captured; + EXPECT_NE(0, server.exitCode()) << server.captured; + EXPECT_NE(server.captured.find("must name its transport provider in the " + "YAML transport section"), + std::string::npos) + << server.captured; +} + // --------------------------------------------------------------------------- // ONE RING PER DECODER, TWO PROCESSES: the server opens one provider // instance (one udp endpoint, one ring, one dispatcher) per decoder and diff --git a/libs/qec/unittests/test_decoders_yaml.cpp b/libs/qec/unittests/test_decoders_yaml.cpp index cc099a859..c98ca575c 100644 --- a/libs/qec/unittests/test_decoders_yaml.cpp +++ b/libs/qec/unittests/test_decoders_yaml.cpp @@ -326,6 +326,11 @@ TEST(DecoderYAMLTest, TransportSectionAndMixedDispatch) { EXPECT_EQ(parsed.transport.device_graph.provider, "hololink"); ASSERT_EQ(parsed.transport.device_graph.args.size(), 1u); EXPECT_EQ(parsed.transport.device_graph.args[0], "--pinned-rings"); + const auto resolved = parsed.transport.resolve_device_graph(); + EXPECT_EQ(resolved.provider, "hololink"); + ASSERT_EQ(resolved.args.size(), 2u); + EXPECT_EQ(resolved.args[0], "--num-slots=8"); + EXPECT_EQ(resolved.args[1], "--pinned-rings"); ASSERT_EQ(parsed.decoders.size(), 2u); EXPECT_EQ(parsed.decoders[0].dispatch, cudaq::qec::decoding::config::DecoderDispatch::host); diff --git a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh index 2df3bf3d9..d44e687be 100755 --- a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh +++ b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh @@ -219,7 +219,7 @@ Run options: --num-shots N Limit number of shots --page-size N Ring buffer slot size in bytes (default: 384) --frame-size N Server TX SGE bytes, cpu_roce only (default: 64; - gpu_roce uses page-size as QEC_DEVICE_GRAPH_FRAME_SIZE) + gpu_roce derives payload size from page-size in YAML) --gpu N GPU device id for gpu_roce (default: 0) --gpu-roce-num-pages N Server GPU RoCE ring pages (default: auto-align; starts from playback window pages) @@ -292,17 +292,24 @@ fi # Some DOCA registrations require the gpu_roce server ring allocation to be # host-page aligned. Keep playback capacity independent from the server ring, # and choose a server page count that satisfies the allocation contract. -if [[ "$TRANSPORT" == "gpu_roce" && "$GPU_ROCE_NUM_PAGES" == "auto" ]]; then +if [[ "$TRANSPORT" == "gpu_roce" ]]; then HOST_PAGE_SIZE=$(getconf PAGESIZE 2>/dev/null || echo 4096) SERVER_PAGE_SIZE=$(( ((PAGE_SIZE + 127) / 128) * 128 )) - GPU_ROCE_NUM_PAGES="$PLAYBACK_NUM_PAGES" - while (( (SERVER_PAGE_SIZE * GPU_ROCE_NUM_PAGES) % HOST_PAGE_SIZE != 0 )); do - ((GPU_ROCE_NUM_PAGES++)) - if (( GPU_ROCE_NUM_PAGES > 65536 )); then - echo "ERROR: unable to auto-align gpu_roce ring for page-size=$PAGE_SIZE host-page-size=$HOST_PAGE_SIZE" >&2 - exit 1 - fi - done + GPU_ROCE_PAYLOAD_SIZE=$((PAGE_SIZE - 24)) # RPCHeader is 24 bytes. + if (( GPU_ROCE_PAYLOAD_SIZE <= 0 )); then + echo "ERROR: gpu_roce page-size must exceed the 24-byte RPC header" >&2 + exit 1 + fi + if [[ "$GPU_ROCE_NUM_PAGES" == "auto" ]]; then + GPU_ROCE_NUM_PAGES="$PLAYBACK_NUM_PAGES" + while (( (SERVER_PAGE_SIZE * GPU_ROCE_NUM_PAGES) % HOST_PAGE_SIZE != 0 )); do + ((GPU_ROCE_NUM_PAGES++)) + if (( GPU_ROCE_NUM_PAGES > 65536 )); then + echo "ERROR: unable to auto-align gpu_roce ring for page-size=$PAGE_SIZE host-page-size=$HOST_PAGE_SIZE" >&2 + exit 1 + fi + done + fi fi # ============================================================================ @@ -981,6 +988,41 @@ extract_hex() { # Start the decoding server against $1=peer_ip $2=remote_qp; scrape its # endpoint line into SERVER_QP / SERVER_RKEY / SERVER_ADDR. +prepare_gpu_roce_config() { + local peer_ip="$1" remote_qp="$2" + + # A caller-supplied transport section is already authoritative. Generated + # decoder-only configs get a temporary launch copy carrying the runtime + # endpoint discovered from the FPGA/emulator; never modify the source file. + if grep -Eq '^[[:space:]]*transport:' "$CONFIG_FILE"; then + _info "Using gpu_roce transport settings from $CONFIG_FILE" + return 0 + fi + + local runtime_config + runtime_config=$(mktemp /tmp/hsb_decoding_server_config.XXXXXX.yml) + TEMP_FILES+=("$runtime_config") + if [[ "$(tail -n 1 "$CONFIG_FILE")" == "..." ]]; then + sed '$d' "$CONFIG_FILE" > "$runtime_config" + else + cp "$CONFIG_FILE" "$runtime_config" + fi + cat >> "$runtime_config" < Date: Tue, 4 Aug 2026 15:20:15 -0700 Subject: [PATCH 2/6] Make GPU RoCE transport YAML-only across dispatch paths Signed-off-by: Melody Ren --- .../cudaq/qec/realtime/decoding_config.h | 18 +-- .../tools/decoding-server/decoding_server.cpp | 95 ++++++++++---- .../realtime/test_decoding_server.cpp | 121 +++++++++++++++++- 3 files changed, 196 insertions(+), 38 deletions(-) diff --git a/libs/qec/include/cudaq/qec/realtime/decoding_config.h b/libs/qec/include/cudaq/qec/realtime/decoding_config.h index f678aa0aa..31beb49f6 100644 --- a/libs/qec/include/cudaq/qec/realtime/decoding_config.h +++ b/libs/qec/include/cudaq/qec/realtime/decoding_config.h @@ -108,7 +108,7 @@ struct transport_shape_override { /// soname is "libcudaq-realtime-bridge-" + name + ".so", with '_' in the /// name mapping to '-' to match the shipped hyphenated sonames (so /// gpu_roce loads libcudaq-realtime-bridge-gpu-roce.so). - /// Empty = inherit the section/CLI default. + /// Empty = inherit the section default. std::string provider; /// Extra provider arguments appended for this shape's rings. std::vector args; @@ -129,11 +129,12 @@ struct transport_shape_override { /// provider: udp # "gpu_roce" on an HSB rig /// args: [--pinned-rings] /// -/// Resolution per ring: shape override (device_graph rings) > this -/// section's provider/args > the server's --transport CLI fallback. The -/// CLI flag only applies when this section names no provider; a config -/// that names one plus an explicit --transport is rejected at startup -/// (the deployment file is the source of truth for the wire). +/// Resolution per ring: shape override (device_graph rings) > this section's +/// provider/args. The standalone server permits a CLI fallback only for +/// non-GPU-RoCE, host-only configurations whose YAML names no provider. A +/// configuration containing device_graph dispatch or using GPU RoCE must +/// define its provider and arguments in YAML (the deployment file is the +/// source of truth for GPU-visible rings). struct transport_config { std::string provider; std::vector args; @@ -158,8 +159,9 @@ struct transport_config { class multi_decoder_config { public: std::vector decoders; - /// Optional server-level transport section (empty provider/args = not - /// specified; the server's CLI defaults apply). + /// Optional server-level transport section. The standalone server's CLI + /// defaults may fill an empty section only for non-GPU-RoCE, host-only + /// configurations. transport_config transport; bool operator==(const multi_decoder_config &) const = default; diff --git a/libs/qec/tools/decoding-server/decoding_server.cpp b/libs/qec/tools/decoding-server/decoding_server.cpp index 2da75ebd4..81c0f2813 100644 --- a/libs/qec/tools/decoding-server/decoding_server.cpp +++ b/libs/qec/tools/decoding-server/decoding_server.cpp @@ -14,9 +14,12 @@ /// Both the decoder and the transport are configuration, not code: /// - decoders come from `--config=` /// (multi_decoder_config::from_yaml_str); -/// - the transport comes from `--transport=`: a CUDA-Q -/// realtime bridge PROVIDER, loaded at runtime through the transport- -/// provider interface (bridge_interface.h). A bare name resolves to +/// - the transport comes from the YAML `transport:` section for every +/// device_graph decoder. Host-only configurations using other providers +/// may instead use `--transport=`. The selected +/// CUDA-Q realtime bridge PROVIDER is loaded at runtime through the +/// transport-provider interface (bridge_interface.h). A bare name resolves +/// to /// `libcudaq-realtime-bridge-.so` next to the CUDA-Q realtime /// libraries, with '_' in the name mapping to '-' to match the shipped /// hyphenated sonames (udp, cpu_roce and gpu_roce ship there); a value @@ -50,6 +53,8 @@ /// decoding_server --config= /// [--transport=] [--timeout=60] /// [provider args, forwarded verbatim...] +/// Configurations containing a device_graph decoder must put the provider and +/// all provider arguments in the YAML transport section. /// /// NOTE: --slot-size must match the caller channel's slot size (each frame /// occupies one full slot stride on both wires). @@ -151,9 +156,12 @@ bool parse_args(int argc, char **argv, ServerConfig &cfg) { "--num-slots=N --slot-size=N --device=NAME " "--local-ip=ADDR --qp_config=rendezvous|hsb_fpga " "--peer-ip=ADDR --remote-qp=N --frame-size=N]\n" - "--transport applies only when the YAML transport " - "section names no provider (a conflict is a startup " - "error).\n" + "--transport and command-line provider arguments apply " + "only to non-gpu_roce, host-only configurations whose " + "YAML transport section names no provider (a conflict is " + "a startup error). Configurations containing device_graph " + "decoders or using gpu_roce must put the provider and its " + "arguments in YAML.\n" "Providers and their args are defined by the installed " "cudaq-realtime (libcudaq-realtime-bridge-.so, " "with '_' in mapping to '-'); " @@ -260,6 +268,15 @@ std::string resolve_provider_lib(const std::string &transport) { return soname; } +bool is_gpu_roce_provider(const std::string &provider) { + const auto slash = provider.find_last_of('/'); + const std::string name = + slash == std::string::npos ? provider : provider.substr(slash + 1); + return name == "gpu_roce" || name == "gpu-roce" || + name == "libcudaq-realtime-bridge-gpu-roce.so" || + name == "libcudaq-realtime-bridge-gpu_roce.so"; +} + // Split a provider endpoint-info line into its port and the remaining // tokens. std::uint16_t split_endpoint_info(const std::string &endpoint_info, @@ -353,6 +370,42 @@ int main(int argc, char **argv) { decoder_config.decoders.end(), [](const auto &d) { return d.dispatch == config::DecoderDispatch::device_graph; }); + const bool has_device_graph = + std::any_of(decoder_config.decoders.begin(), + decoder_config.decoders.end(), [](const auto &d) { + return d.dispatch == config::DecoderDispatch::device_graph; + }); + const bool has_host = !all_device_graph; + const auto resolved_device_graph = + decoder_config.transport.resolve_device_graph(); + const std::string resolved_host_provider = + decoder_config.transport.provider.empty() + ? cfg.transport + : decoder_config.transport.provider; + const bool host_uses_gpu_roce = + has_host && is_gpu_roce_provider(resolved_host_provider); + // device_graph providers own GPU-visible rings and must be reproducible from + // the deployment YAML in every server topology. Generic CLI provider + // arguments are intentionally rejected rather than guessed to belong to a + // host ring in a mixed-dispatch configuration. + if (has_device_graph && resolved_device_graph.provider.empty()) { + std::cerr << "ERROR: a config containing device_graph dispatch must name " + "its transport provider in the YAML transport section" + << std::endl; + return 1; + } + if (host_uses_gpu_roce && decoder_config.transport.provider.empty()) { + std::cerr << "ERROR: gpu_roce transport must be selected and configured " + "in the YAML transport section" + << std::endl; + return 1; + } + if ((has_device_graph || host_uses_gpu_roce) && !cfg.provider_args.empty()) { + std::cerr << "ERROR: provider arguments for device_graph or gpu_roce " + "transport must be set in the YAML transport section" + << std::endl; + return 1; + } // [2a] device_graph dispatch takes a different shape (device-side // scheduler): bypass the CQR DeviceCallService / HOST_CALL dispatcher and @@ -373,12 +426,6 @@ int main(int argc, char **argv) { // // The standalone device-graph transport has no CLI or environment // fallback: the deployment YAML is its single source of truth. - if (decoder_config.transport.resolve_device_graph().provider.empty()) { - std::cerr << "ERROR: an all-device_graph config must name its transport " - "provider in the YAML transport section" - << std::endl; - return 1; - } try { cudaq::qec::decoding_server::DecodingServer server(cfg.config_path); // QP/rkey/buf already printed to stdout by launch_scheduler() so the @@ -468,11 +515,6 @@ int main(int argc, char **argv) { ? cfg.transport : decoder_config.transport.provider; - std::vector provider_argv; - provider_argv.reserve(cfg.provider_args.size()); - for (auto &a : cfg.provider_args) - provider_argv.push_back(a.data()); - struct DecoderRing { std::int64_t decoder_id = 0; bool device_graph = false; @@ -529,10 +571,10 @@ int main(int argc, char **argv) { // The wire is deployment config, resolved from the YAML's top-level // `transport:` section (never from decoder entries). Per-ring - // resolution: the section's dispatch-shape override (device_graph - // rings) > the section's provider/args > the --transport CLI fallback - // (which only applies when the YAML names no provider -- a conflict is - // rejected at startup above). + // resolution: the section's dispatch-shape override (device_graph rings) + // > the section's provider/args. Host-only configurations may fall back + // to --transport when the YAML names no provider; device_graph rings are + // validated above to be fully YAML-configured. // Every provider name/path resolves the same way --transport does; the // bridge loader caches libraries per name, so different rings may load // different provider libraries in one process. @@ -541,13 +583,18 @@ int main(int argc, char **argv) { std::vector ring_extra_args; if (ring.device_graph) { const auto resolved = transport_section.resolve_device_graph(); - if (!resolved.provider.empty()) - ring_provider_name = resolved.provider; + ring_provider_name = resolved.provider; ring_extra_args = std::move(resolved.args); } else ring_extra_args = transport_section.args; const std::string ring_lib = resolve_provider_lib(ring_provider_name); - std::vector ring_argv = provider_argv; + std::vector ring_argv; + if (!ring.device_graph) { + ring_argv.reserve(cfg.provider_args.size() + ring_extra_args.size()); + for (auto &a : cfg.provider_args) + ring_argv.push_back(a.data()); + } else + ring_argv.reserve(ring_extra_args.size()); for (auto &a : ring_extra_args) ring_argv.push_back(a.data()); diff --git a/libs/qec/unittests/realtime/test_decoding_server.cpp b/libs/qec/unittests/realtime/test_decoding_server.cpp index 55c5bb1bf..66178095d 100644 --- a/libs/qec/unittests/realtime/test_decoding_server.cpp +++ b/libs/qec/unittests/realtime/test_decoding_server.cpp @@ -197,14 +197,15 @@ std::string server_dir() { // and collects its stdout (for the shutdown dispatch-count line). class ServerProcess { public: - // `transport_cli` = false launches the server WITHOUT --transport (and - // without the cpu_roce endpoint args), exercising configs whose wire is - // named by the YAML transport section instead of the command line. + // `transport_cli` = false launches the server WITHOUT --transport or its + // default provider args, exercising configs whose wire is named and + // configured by the YAML transport section instead of the command line. // `capture_stderr` folds the server's stderr into `captured` (used by the // conflict-rejection test to see the startup error). bool start(const std::string &config_file, std::string &error, int ready_timeout_ms = 15000, bool transport_cli = true, - bool capture_stderr = false) { + bool capture_stderr = false, + const std::vector &extra_provider_args = {}) { int out_pipe[2] = {-1, -1}; if (::pipe(out_pipe) != 0) { error = "pipe() failed"; @@ -233,8 +234,10 @@ class ServerProcess { env_or("CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE", "mlx5_0")); args.push_back("--local-ip=" + env_or("CUDAQ_CPU_ROCE_TEST_DAEMON_IP", "10.0.0.2")); + args.push_back("--port=0"); } - args.push_back("--port=0"); + args.insert(args.end(), extra_provider_args.begin(), + extra_provider_args.end()); args.push_back("--timeout=60"); std::vector argv; for (auto &a : args) @@ -572,7 +575,7 @@ TEST(DecodingServerTwoProcess, TwoProcessHostDispatchYamlTransportSection) { std::ofstream config_file(config_path); config_file << "transport:\n" << " provider: udp\n" - << " args: [--num-slots=8]\n" + << " args: [--num-slots=8, --port=0]\n" << "decoders:\n"; for (int id = 0; id < 2; ++id) { config_file << " - id: " << id << "\n" @@ -683,6 +686,112 @@ TEST(DecodingServerTwoProcess, DeviceGraphRequiresYamlTransportProvider) { << server.captured; } +// A mixed-dispatch topology must not obtain its device_graph provider from the +// generic CLI fallback. This is the composed per-ring path rather than the +// standalone all-device_graph path covered above. +TEST(DecodingServerTwoProcess, + MixedDispatchRequiresYamlDeviceGraphTransportProvider) { + const std::string config_path = + ::testing::TempDir() + "/decoding_server_mixed_no_transport.yaml"; + { + std::ofstream config_file(config_path); + config_file << "decoders:\n" + << " - id: 0\n" + << " type: single_error_lut\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n" + << " - id: 1\n" + << " type: single_error_lut\n" + << " dispatch: device_graph\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n"; + } + + ServerProcess server; + std::string error; + EXPECT_FALSE(server.start(config_path, error, 8000, + /*transport_cli=*/true, + /*capture_stderr=*/true)) + << "server unexpectedly reached READY: " << server.captured; + EXPECT_NE(0, server.exitCode()) << server.captured; + EXPECT_NE(server.captured.find("device_graph dispatch must name its " + "transport provider in the YAML"), + std::string::npos) + << server.captured; +} + +// Even when YAML names gpu_roce, generic provider arguments must not create a +// second configuration path into a device_graph ring. +TEST(DecodingServerTwoProcess, DeviceGraphRejectsCliProviderArguments) { + const std::string config_path = + ::testing::TempDir() + "/decoding_server_device_graph_cli_args.yaml"; + { + std::ofstream config_file(config_path); + config_file << "transport:\n" + << " provider: gpu_roce\n" + << " args: [--device=mlx5_yaml]\n" + << "decoders:\n" + << " - id: 0\n" + << " type: single_error_lut\n" + << " dispatch: device_graph\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n"; + } + + ServerProcess server; + std::string error; + EXPECT_FALSE(server.start(config_path, error, 8000, + /*transport_cli=*/false, + /*capture_stderr=*/true, {"--device=mlx5_cli"})) + << "server unexpectedly reached READY: " << server.captured; + EXPECT_NE(0, server.exitCode()) << server.captured; + EXPECT_NE(server.captured.find("provider arguments for device_graph " + "or gpu_roce transport must be set in the " + "YAML"), + std::string::npos) + << server.captured; +} + +// GPU RoCE remains YAML-only even if a caller tries to select it for the +// generic host-dispatch path. +TEST(DecodingServerTwoProcess, HostDispatchRejectsCliGpuRoceTransport) { + ScopedEnv transport("QEC_DECODING_SERVER_TRANSPORT", "gpu_roce"); + const std::string config_path = + ::testing::TempDir() + "/decoding_server_host_cli_gpu_roce.yaml"; + { + std::ofstream config_file(config_path); + config_file << "decoders:\n" + << " - id: 0\n" + << " type: single_error_lut\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n"; + } + + ServerProcess server; + std::string error; + EXPECT_FALSE(server.start(config_path, error, 8000, + /*transport_cli=*/true, + /*capture_stderr=*/true)) + << "server unexpectedly reached READY: " << server.captured; + EXPECT_NE(0, server.exitCode()) << server.captured; + EXPECT_NE(server.captured.find("gpu_roce transport must be selected and " + "configured in the YAML"), + std::string::npos) + << server.captured; +} + // --------------------------------------------------------------------------- // ONE RING PER DECODER, TWO PROCESSES: the server opens one provider // instance (one udp endpoint, one ring, one dispatcher) per decoder and From 10cd4cb9eda387c20855f3ef243398eef969a1cd Mon Sep 17 00:00:00 2001 From: Melody Ren Date: Wed, 5 Aug 2026 12:02:27 -0700 Subject: [PATCH 3/6] Fix GPU RoCE YAML configuration paths Signed-off-by: Melody Ren --- .../run_realtime_decoding.sh | 94 +++++++++++++------ .../DeviceGraphTransceiver.cpp | 9 ++ .../tools/decoding-server/decoding_server.cpp | 35 +++++-- .../realtime/test_decoding_server.cpp | 65 +++++++++++++ 4 files changed, 168 insertions(+), 35 deletions(-) diff --git a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh index 1381185ad..29452c7ed 100755 --- a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh +++ b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh @@ -27,7 +27,7 @@ # FPGA into the server's RDMA ring (needs a ConnectX NIC # cabled to the FPGA). CPU decoders ride the cpu_roce # wire on host dispatch; nv-qldpc defaults to the -# hololink wire on device_graph dispatch (the GPU +# gpu_roce wire on device_graph dispatch (the GPU # device-call scheduler). There is NO emulator here -- # emulator testing lives in the unittests # hsb_fpga_decoding_server_test.sh. @@ -117,7 +117,7 @@ SPACING="10" # DISPATCH which engine consumes each decoder's ring: host (a CPU dispatcher # thread) or device_graph (the GPU device-call scheduler) # The combination gate below rejects pairings this example has not wired up. -WIRE="" # udp | cpu_roce | hololink +WIRE="" # udp | cpu_roce | gpu_roce DISPATCH="" # host | device_graph GPU_ID=0 @@ -187,9 +187,9 @@ Common: --num-shots N Shots (default: 85 fpga / 200 qpu-kernel) --gpu N GPU id for nv-qldpc (default 0) --wire W Bridge provider carrying syndromes into the server: - udp | cpu_roce | hololink (default: derived -- + udp | cpu_roce | gpu_roce (default: derived -- qpu-kernel -> udp; fpga -> cpu_roce for host - dispatch, hololink for device_graph) + dispatch, gpu_roce for device_graph) --dispatch D Per-decoder ring consumer: host | device_graph (default: derived -- fpga + nv-qldpc-decoder -> device_graph, everything else -> host) @@ -270,7 +270,7 @@ fi # The bridge NIC and the FPGA must share a /24: there is no gateway on the # FPGA link, and --setup-network flushes the NIC and assigns BRIDGE_IP/24, so # a mismatched pair leaves the FPGA unreachable -- the run then dies minutes -# later as an opaque hololink read_timeout_error. Reject it up front. +# later as an opaque HSB read timeout. Reject it up front. if [[ "$SOURCE" == "fpga" && "${BRIDGE_IP%.*}" != "${FPGA_IP%.*}" ]]; then echo "ERROR: --bridge-ip $BRIDGE_IP and --fpga-ip $FPGA_IP are on different" >&2 echo " /24 subnets; the FPGA data path is same-subnet routed." >&2 @@ -332,7 +332,7 @@ fi # qpu-kernel : udp wire, host dispatch (any decoder incl. trt_decoder) # qpu-kernel : cpu_roce wire, host dispatch (any decoder incl. trt_decoder) # fpga : cpu_roce wire, host dispatch (any decoder incl. trt_decoder) -# fpga : hololink wire, device_graph dispatch (nv-qldpc-decoder) +# fpga : gpu_roce wire, device_graph dispatch (nv-qldpc-decoder) # Everything else is a real configuration of the decoding server that this # example does not (yet) exercise, so it is rejected with the reason. # --------------------------------------------------------------------------- @@ -359,18 +359,19 @@ if [[ "$SOURCE" == "fpga" && "$DECODER" == "nv-qldpc-decoder" ]] && \ fi [[ "$WIRE" == "cpu-roce" ]] && WIRE="cpu_roce" # accept both spellings +[[ "$WIRE" == "gpu-roce" ]] && WIRE="gpu_roce" if [[ -z "$WIRE" ]]; then if [[ "$SOURCE" == "qpu-kernel" ]]; then WIRE="udp" - elif [[ "$DISPATCH" == "device_graph" ]]; then WIRE="hololink" + elif [[ "$DISPATCH" == "device_graph" ]]; then WIRE="gpu_roce" else WIRE="cpu_roce"; fi fi -case "$WIRE" in udp|cpu_roce|hololink) ;; - *) echo "ERROR: --wire must be udp, cpu_roce, or hololink (got '$WIRE')" >&2; exit 1 ;; +case "$WIRE" in udp|cpu_roce|gpu_roce) ;; + *) echo "ERROR: --wire must be udp, cpu_roce, or gpu_roce (got '$WIRE')" >&2; exit 1 ;; esac if [[ "$SOURCE" == "qpu-kernel" ]]; then - if [[ "$WIRE" == "hololink" ]]; then - echo "ERROR: the qpu-kernel source is not wired to the hololink wire in" >&2 + if [[ "$WIRE" == "gpu_roce" ]]; then + echo "ERROR: the qpu-kernel source is not wired to the gpu_roce wire in" >&2 echo " this example (use --wire udp or --wire cpu_roce)." >&2; exit 1 fi if [[ "$DISPATCH" != "host" ]]; then @@ -383,8 +384,8 @@ else echo "ERROR: device_graph dispatch serves only nv-qldpc-decoder" >&2 echo " (got '$DECODER'); CPU decoders use --dispatch host." >&2; exit 1 fi - if [[ "$WIRE" != "hololink" ]]; then - echo "ERROR: device_graph dispatch on the FPGA requires the hololink" >&2 + if [[ "$WIRE" != "gpu_roce" ]]; then + echo "ERROR: device_graph dispatch on the FPGA requires the gpu_roce" >&2 echo " wire (got '$WIRE')." >&2; exit 1 fi else @@ -395,11 +396,9 @@ else fi fi -# The provider library soname is hyphenated (libcudaq-realtime-bridge-cpu-roce.so) -# while the conventional token is cpu_roce; the server's resolver composes the -# soname literally from the token, so pass the hyphenated form on the wire. +# Provider names use the YAML spelling; the server maps underscores to the +# installed providers' hyphenated sonames. WIRE_TOKEN="$WIRE" -[[ "$WIRE_TOKEN" == "cpu_roce" ]] && WIRE_TOKEN="cpu-roce" # --------------------------------------------------------------------------- # qpu-kernel over cpu_roce: RDMA topology. Same four-env-var convention as @@ -474,7 +473,7 @@ resolve_paths() { _err "lowered kernel not found: $KERNEL_BIN (build the example, or --kernel PATH)"; exit 1 fi if [[ "$SOURCE" == "fpga" && ! -x "$PLAYBACK_BIN" ]]; then - _err "playback tool not found: $PLAYBACK_BIN (a hololink-enabled deliverable)"; exit 1 + _err "playback tool not found: $PLAYBACK_BIN (an HSB-enabled deliverable)"; exit 1 fi # Load path: deliverable libs + plugins, plus the CUDA-Q runtime/realtime. @@ -939,20 +938,59 @@ setup_network_cpu_roce() { fi } +prepare_gpu_roce_config() { + local peer_ip="$1" remote_qp="$2" num_pages="$3" + local remote_qp_decimal=$((remote_qp)) + local payload_size=$((PAGE_SIZE - 24)) + if (( payload_size <= 0 )); then + _err "gpu_roce page size ($PAGE_SIZE) must exceed the 24-byte RPC header" + return 1 + fi + if grep -Eq '^[[:space:]]*transport:' "$CONFIG_FILE"; then + _err "generated config already contains a transport section: $CONFIG_FILE" + return 1 + fi + + # CONFIG_FILE already lives in this run's temporary directory. Strip the + # generator's terminal YAML document marker, then serialize the dynamic + # endpoint once so the server sees one configuration source. + awk ' + { lines[NR] = $0 } + END { + last = NR + while (last > 0 && lines[last] ~ /^[[:space:]]*$/) + --last + if (last > 0 && lines[last] ~ /^[[:space:]]*\.\.\.[[:space:]]*$/) + --last + for (line = 1; line <= last; ++line) + print lines[line] + } + ' "$CONFIG_FILE" > "$CONFIG_FILE.tmp" && mv "$CONFIG_FILE.tmp" "$CONFIG_FILE" + cat >> "$CONFIG_FILE" < >(tee "$server_log") 2>&1 & ready="QEC_DECODING_SERVER_READY device_graph" @@ -1012,7 +1050,7 @@ run_fpga() { : "${BRIDGE_DEVICE:=${IB_DEVICE:-rocep1s0f0}}" local server_log="$GEN_DIR/server.log" - start_roce_server "$FPGA_IP" "0x2" "$server_log" || return 1 + start_roce_server "$FPGA_IP" "0x2" "$server_log" "$DEVICE_GRAPH_NUM_PAGES" || return 1 _log "Streaming syndromes from the FPGA via playback (spacing=${SPACING}us)" # The FPGA writes syndrome frame rid to RDMA slot (rid % num-pages), so the @@ -1023,7 +1061,7 @@ run_fpga() { # device_graph ring is DEVICE_GRAPH_NUM_PAGES. local pb_pages="$NUM_SLOTS" if [[ "$DISPATCH" == "device_graph" ]]; then pb_pages="$DEVICE_GRAPH_NUM_PAGES"; fi - local args=( --hololink "$FPGA_IP" --per-round --config "$CONFIG_FILE" + local args=( --hsb-ip "$FPGA_IP" --per-round --config "$CONFIG_FILE" --syndromes "$SYNDROMES_FILE" --qp-number "$SERVER_QP" --rkey "$SERVER_RKEY" --buffer-addr "$SERVER_ADDR" --page-size "$PAGE_SIZE" --num-pages "$pb_pages" ) $VERIFY && args+=(--verify) diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp index f69fe8543..4af4a2b83 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DeviceGraphTransceiver.cpp @@ -51,6 +51,10 @@ std::string resolve_provider_library(const std::string &provider) { return soname; } +bool is_gpu_argument(const std::string &arg) { + return arg == "--gpu" || arg.rfind("--gpu=", 0) == 0; +} + } // namespace // --------------------------------------------------------------------------- @@ -63,6 +67,11 @@ DeviceGraphTransceiver::DeviceGraphTransceiver(const DeviceGraphConfig &config) throw std::runtime_error( "DeviceGraphTransceiver: device_graph transport provider must be set " "in YAML"); + if (std::any_of(config.provider_args.begin(), config.provider_args.end(), + is_gpu_argument)) + throw std::runtime_error( + "DeviceGraphTransceiver: transport arguments must not set --gpu; " + "set decoder cuda_device_id in YAML instead"); // Bring the GpuRoceTransceiver up through the bridge-provider interface: // create() = gpu_roce_create_transceiver + gpu_roce_start (3-kernel shape: diff --git a/libs/qec/tools/decoding-server/decoding_server.cpp b/libs/qec/tools/decoding-server/decoding_server.cpp index 81c0f2813..65e0be8b2 100644 --- a/libs/qec/tools/decoding-server/decoding_server.cpp +++ b/libs/qec/tools/decoding-server/decoding_server.cpp @@ -277,6 +277,10 @@ bool is_gpu_roce_provider(const std::string &provider) { name == "libcudaq-realtime-bridge-gpu_roce.so"; } +bool is_gpu_argument(const std::string &arg) { + return arg == "--gpu" || starts_with(arg, "--gpu="); +} + // Split a provider endpoint-info line into its port and the remaining // tokens. std::uint16_t split_endpoint_info(const std::string &endpoint_info, @@ -406,6 +410,14 @@ int main(int argc, char **argv) { << std::endl; return 1; } + if (has_device_graph && + std::any_of(resolved_device_graph.args.begin(), + resolved_device_graph.args.end(), is_gpu_argument)) { + std::cerr << "ERROR: device_graph transport arguments must not set --gpu; " + "set decoder cuda_device_id in YAML instead" + << std::endl; + return 1; + } // [2a] device_graph dispatch takes a different shape (device-side // scheduler): bypass the CQR DeviceCallService / HOST_CALL dispatcher and @@ -588,14 +600,23 @@ int main(int argc, char **argv) { } else ring_extra_args = transport_section.args; const std::string ring_lib = resolve_provider_lib(ring_provider_name); - std::vector ring_argv; + // Provider parsers follow the C argv convention and start at argv[1]. + // Keep the strings alive until create() returns, add a program-name + // placeholder, and only then form the char-pointer view. For a + // device_graph ring, cuda_device_id is the sole GPU-placement setting. + std::vector ring_args{"decoding-server-transport"}; if (!ring.device_graph) { - ring_argv.reserve(cfg.provider_args.size() + ring_extra_args.size()); - for (auto &a : cfg.provider_args) - ring_argv.push_back(a.data()); - } else - ring_argv.reserve(ring_extra_args.size()); - for (auto &a : ring_extra_args) + ring_args.insert(ring_args.end(), cfg.provider_args.begin(), + cfg.provider_args.end()); + } + ring_args.insert(ring_args.end(), ring_extra_args.begin(), + ring_extra_args.end()); + if (ring.device_graph) + ring_args.push_back("--gpu=" + std::to_string(ring.gpu_id)); + + std::vector ring_argv; + ring_argv.reserve(ring_args.size()); + for (auto &a : ring_args) ring_argv.push_back(a.data()); if (cudaq_bridge_create_from_library(&ring.bridge, ring_lib.c_str(), diff --git a/libs/qec/unittests/realtime/test_decoding_server.cpp b/libs/qec/unittests/realtime/test_decoding_server.cpp index 66178095d..c31b14d41 100644 --- a/libs/qec/unittests/realtime/test_decoding_server.cpp +++ b/libs/qec/unittests/realtime/test_decoding_server.cpp @@ -615,6 +615,35 @@ TEST(DecodingServerTwoProcess, TwoProcessHostDispatchYamlTransportSection) { EXPECT_GE(dispatched, 6) << "server output:\n" << server.captured; } +// Provider parsers start at argv[1]. A malformed first YAML argument must be +// observed and rejected rather than silently becoming the program name. +TEST(DecodingServerTwoProcess, YamlProviderParsesFirstArgument) { + const std::string config_path = + ::testing::TempDir() + "/decoding_server_yaml_first_arg.yaml"; + { + std::ofstream config_file(config_path); + config_file << "transport:\n" + << " provider: udp\n" + << " args: [--port=not-a-port]\n" + << "decoders:\n" + << " - id: 0\n" + << " type: single_error_lut\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n"; + } + + ServerProcess server; + std::string error; + EXPECT_FALSE(server.start(config_path, error, 8000, + /*transport_cli=*/false, + /*capture_stderr=*/true)) + << "server unexpectedly reached READY: " << server.captured; + EXPECT_NE(0, server.exitCode()) << server.captured; +} + // A YAML that names its provider cannot be contradicted from the command // line: --transport alongside a transport section is a startup error, not a // silent precedence decision. @@ -761,6 +790,42 @@ TEST(DecodingServerTwoProcess, DeviceGraphRejectsCliProviderArguments) { << server.captured; } +// GPU placement belongs to decoder.cuda_device_id. Allowing --gpu in the +// transport arguments would put two contradictory settings in the same YAML. +TEST(DecodingServerTwoProcess, DeviceGraphRejectsTransportGpuArgument) { + const std::string config_path = + ::testing::TempDir() + "/decoding_server_device_graph_gpu_arg.yaml"; + { + std::ofstream config_file(config_path); + config_file << "transport:\n" + << " provider: gpu_roce\n" + << " device_graph:\n" + << " args: [--gpu=1]\n" + << "decoders:\n" + << " - id: 0\n" + << " type: single_error_lut\n" + << " dispatch: device_graph\n" + << " cuda_device_id: 1\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n"; + } + + ServerProcess server; + std::string error; + EXPECT_FALSE(server.start(config_path, error, 8000, + /*transport_cli=*/false, + /*capture_stderr=*/true)) + << "server unexpectedly reached READY: " << server.captured; + EXPECT_NE(0, server.exitCode()) << server.captured; + EXPECT_NE(server.captured.find("must not set --gpu; set decoder " + "cuda_device_id in YAML instead"), + std::string::npos) + << server.captured; +} + // GPU RoCE remains YAML-only even if a caller tries to select it for the // generic host-dispatch path. TEST(DecodingServerTwoProcess, HostDispatchRejectsCliGpuRoceTransport) { From 8299436c0a30b6f1c126c0929a718b59c3dfdc6b Mon Sep 17 00:00:00 2001 From: Melody Ren Date: Wed, 12 Aug 2026 13:29:48 -0700 Subject: [PATCH 4/6] Make GPU RoCE YAML authoritative across helper paths Signed-off-by: Melody Ren --- .../tools/decoding-server/decoding_server.cpp | 26 ++- .../realtime/test_decoding_server.cpp | 36 ++++ .../utils/hsb_fpga_decoding_server_test.sh | 184 ++++++++++++------ 3 files changed, 179 insertions(+), 67 deletions(-) diff --git a/libs/qec/tools/decoding-server/decoding_server.cpp b/libs/qec/tools/decoding-server/decoding_server.cpp index 65e0be8b2..e510f13fe 100644 --- a/libs/qec/tools/decoding-server/decoding_server.cpp +++ b/libs/qec/tools/decoding-server/decoding_server.cpp @@ -410,14 +410,26 @@ int main(int argc, char **argv) { << std::endl; return 1; } - if (has_device_graph && + const bool device_graph_sets_gpu = + has_device_graph && std::any_of(resolved_device_graph.args.begin(), - resolved_device_graph.args.end(), is_gpu_argument)) { + resolved_device_graph.args.end(), is_gpu_argument); + const bool host_gpu_roce_sets_gpu = + host_uses_gpu_roce && + std::any_of(decoder_config.transport.args.begin(), + decoder_config.transport.args.end(), is_gpu_argument); + if (device_graph_sets_gpu) { std::cerr << "ERROR: device_graph transport arguments must not set --gpu; " "set decoder cuda_device_id in YAML instead" << std::endl; return 1; } + if (host_gpu_roce_sets_gpu) { + std::cerr << "ERROR: gpu_roce transport arguments must not set --gpu; set " + "decoder cuda_device_id in YAML instead" + << std::endl; + return 1; + } // [2a] device_graph dispatch takes a different shape (device-side // scheduler): bypass the CQR DeviceCallService / HOST_CALL dispatcher and @@ -578,8 +590,7 @@ int main(int argc, char **argv) { const auto &dc = decoder_config.decoders[i]; ring.decoder_id = dc.id; ring.device_graph = (dc.dispatch == config::DecoderDispatch::device_graph); - if (ring.device_graph) - ring.gpu_id = dc.cuda_device_id.value_or(0); + ring.gpu_id = dc.cuda_device_id.value_or(0); // The wire is deployment config, resolved from the YAML's top-level // `transport:` section (never from decoder entries). Per-ring @@ -599,11 +610,12 @@ int main(int argc, char **argv) { ring_extra_args = std::move(resolved.args); } else ring_extra_args = transport_section.args; + const bool ring_uses_gpu_roce = is_gpu_roce_provider(ring_provider_name); const std::string ring_lib = resolve_provider_lib(ring_provider_name); // Provider parsers follow the C argv convention and start at argv[1]. // Keep the strings alive until create() returns, add a program-name - // placeholder, and only then form the char-pointer view. For a - // device_graph ring, cuda_device_id is the sole GPU-placement setting. + // placeholder, and only then form the char-pointer view. For a gpu_roce + // ring, cuda_device_id is the sole GPU-placement setting. std::vector ring_args{"decoding-server-transport"}; if (!ring.device_graph) { ring_args.insert(ring_args.end(), cfg.provider_args.begin(), @@ -611,7 +623,7 @@ int main(int argc, char **argv) { } ring_args.insert(ring_args.end(), ring_extra_args.begin(), ring_extra_args.end()); - if (ring.device_graph) + if (ring.device_graph || ring_uses_gpu_roce) ring_args.push_back("--gpu=" + std::to_string(ring.gpu_id)); std::vector ring_argv; diff --git a/libs/qec/unittests/realtime/test_decoding_server.cpp b/libs/qec/unittests/realtime/test_decoding_server.cpp index ff5417c6a..8538aba2c 100644 --- a/libs/qec/unittests/realtime/test_decoding_server.cpp +++ b/libs/qec/unittests/realtime/test_decoding_server.cpp @@ -750,6 +750,42 @@ TEST(DecodingServerTwoProcess, DeviceGraphRejectsTransportGpuArgument) { << server.captured; } +// Host dispatch over gpu_roce has the same single source of GPU placement: +// decoder.cuda_device_id. Reject the duplicate transport setting before the +// provider is loaded so this regression needs no GPU or RoCE device. +TEST(DecodingServerTwoProcess, HostGpuRoceRejectsTransportGpuArgument) { + const std::string config_path = + ::testing::TempDir() + "/decoding_server_host_gpu_roce_gpu_arg.yaml"; + { + std::ofstream config_file(config_path); + config_file << "transport:\n" + << " provider: gpu_roce\n" + << " args: [--gpu=1]\n" + << "decoders:\n" + << " - id: 0\n" + << " type: single_error_lut\n" + << " cuda_device_id: 1\n" + << " block_size: 3\n" + << " syndrome_size: 3\n" + << " H_sparse: [0, -1, 1, -1, 2, -1]\n" + << " O_sparse: [0, -1, 1, -1, 2, -1]\n" + << " D_sparse: [0, -1, 1, -1, 2, -1]\n"; + } + + ServerProcess server; + std::string error; + EXPECT_FALSE(server.start(config_path, error, 8000, + /*transport_cli=*/false, + /*capture_stderr=*/true)) + << "server unexpectedly reached READY: " << server.captured; + EXPECT_NE(0, server.exitCode()) << server.captured; + EXPECT_NE(server.captured.find("gpu_roce transport arguments must not set " + "--gpu; set decoder cuda_device_id in YAML " + "instead"), + std::string::npos) + << server.captured; +} + // GPU RoCE remains YAML-only even if a caller tries to select it for the // generic host-dispatch path. TEST(DecodingServerTwoProcess, HostDispatchRejectsCliGpuRoceTransport) { diff --git a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh index 336d0bed7..d1502bb5e 100755 --- a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh +++ b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh @@ -898,27 +898,6 @@ generate_data_files() { _info "Config carries the trt_decoder entry (onnx_load_path set)" fi - # The server selects the dispatch shape from the per-decoder `dispatch:` - # YAML key (default host). For device_graph, `cuda_device_id` pins graph - # capture and worker-thread execution to the selected GPU. The generator - # doesn't emit these non-default optional fields, so inject them into our - # generated config directly under the decoder's `type:` line. - if [[ "$TRANSPORT" == "gpu_roce" ]]; then - _info "Injecting 'dispatch: device_graph' and cuda_device_id=$GPU_ID into $(basename "$CONFIG_FILE")" - awk -v gpu_id="$GPU_ID" '{ print } - /^[[:space:]]*type:/ && !done { - print " dispatch: device_graph" - print " cuda_device_id: " gpu_id - done = 1 - }' "$CONFIG_FILE" > "${CONFIG_FILE}.tmp" \ - && mv "${CONFIG_FILE}.tmp" "$CONFIG_FILE" - if ! grep -q "dispatch:.*device_graph" "$CONFIG_FILE" || \ - ! grep -q "cuda_device_id:.*$GPU_ID" "$CONFIG_FILE"; then - _err "Failed to inject device_graph dispatch/cuda_device_id into $CONFIG_FILE" - return 1 - fi - fi - _info "$GENERATOR_BIN --distance $GEN_DISTANCE --num_rounds $GEN_ROUNDS" \ "--p_spam $GEN_P_SPAM --num_shots $GEN_SHOTS --yaml $(basename "$CONFIG_FILE")" \ "--save_syndrome $(basename "$SYNDROMES_FILE")" @@ -1059,46 +1038,132 @@ extract_hex() { # endpoint line into SERVER_QP / SERVER_RKEY / SERVER_ADDR. prepare_gpu_roce_config() { local peer_ip="$1" remote_qp="$2" + local source_config="$CONFIG_FILE" runtime_config - # A caller-supplied transport section is already authoritative. Generated - # decoder-only configs get a temporary launch copy carrying the runtime - # endpoint discovered from the FPGA/emulator; never modify the source file. - if grep -Eq '^[[:space:]]*transport:' "$CONFIG_FILE"; then - _info "Using gpu_roce transport settings from $CONFIG_FILE" - return 0 + if ! python3 -c 'import yaml' 2>/dev/null; then + _err "python3 module 'yaml' (PyYAML) is required to prepare the gpu_roce launch config" + return 1 fi - - local runtime_config runtime_config=$(mktemp /tmp/hsb_decoding_server_config.XXXXXX.yml) TEMP_FILES+=("$runtime_config") - # Strip a terminal YAML document marker even when blank lines follow it; - # the generated transport section must remain in the same document. - awk ' - { lines[NR] = $0 } - END { - last = NR - while (last > 0 && lines[last] ~ /^[[:space:]]*$/) - --last - if (last > 0 && lines[last] ~ /^[[:space:]]*\.\.\.[[:space:]]*$/) - --last - for (line = 1; line <= last; ++line) - print lines[line] - } - ' "$CONFIG_FILE" > "$runtime_config" - cat >> "$runtime_config" <= len(args) or args[index + 1].startswith("--"): + raise ValueError(f"{location}.args has no value for {option}") + index += 2 + else: + index += 1 + return result + + +source_path, output_path, gpu_text, device, peer_ip, remote_qp, page_size, \ + num_pages, payload_size = sys.argv[1:] + +try: + try: + gpu_id = int(gpu_text, 10) + except ValueError as error: + raise ValueError( + f"requested cuda_device_id {gpu_text!r} is not an integer") from error + if gpu_id < 0: + raise ValueError("requested cuda_device_id must be non-negative") + + with open(source_path, encoding="utf-8") as source: + config = yaml.safe_load(source) + if not isinstance(config, dict): + raise ValueError("configuration root must be a YAML mapping") + + decoders = config.get("decoders") + if not isinstance(decoders, list) or len(decoders) != 1: + count = len(decoders) if isinstance(decoders, list) else "non-list" + raise ValueError( + "gpu_roce HSB helper supports exactly one decoder " + f"(found {count})") + decoder = decoders[0] + if not isinstance(decoder, dict): + raise ValueError("decoders[0] must be a YAML mapping") + + if "dispatch" not in decoder: + decoder["dispatch"] = "device_graph" + elif decoder["dispatch"] != "device_graph": + raise ValueError( + "decoders[0].dispatch must be device_graph for gpu_roce " + f"(found {decoder['dispatch']!r})") + + if "cuda_device_id" not in decoder: + decoder["cuda_device_id"] = gpu_id + elif (type(decoder["cuda_device_id"]) is not int or + decoder["cuda_device_id"] != gpu_id): + raise ValueError( + "decoders[0].cuda_device_id contradicts --gpu " + f"(YAML {decoder['cuda_device_id']!r}, requested {gpu_id})") + + transport = config.get("transport", {}) + if not isinstance(transport, dict): + raise ValueError("transport must be a YAML mapping") + if transport.get("provider", "gpu_roce") != "gpu_roce": + raise ValueError( + "transport.provider must be gpu_roce " + f"(found {transport['provider']!r})") + device_graph = transport.get("device_graph", {}) + if not isinstance(device_graph, dict): + raise ValueError("transport.device_graph must be a YAML mapping") + if device_graph.get("provider", "gpu_roce") != "gpu_roce": + raise ValueError( + "transport.device_graph.provider must be gpu_roce " + f"(found {device_graph['provider']!r})") + args = launch_args(transport, "transport") + args += launch_args(device_graph, "transport.device_graph") + config["transport"] = {"provider": "gpu_roce", "args": args + [ + f"--device={device}", + f"--peer-ip={peer_ip}", + f"--remote-qp={remote_qp}", + f"--page-size={page_size}", + f"--num-pages={num_pages}", + f"--payload-size={payload_size}", + ]} + + with open(output_path, "w", encoding="utf-8") as output: + yaml.safe_dump(config, output, explicit_end=True, sort_keys=False) +except (OSError, TypeError, ValueError, yaml.YAMLError) as error: + print(f"ERROR: invalid gpu_roce launch config {source_path}: {error}", + file=sys.stderr) + sys.exit(1) +PY + then + return 1 + fi CONFIG_FILE="$runtime_config" - _info "GPU RoCE launch settings written to YAML: $CONFIG_FILE" + _info "Authoritative GPU RoCE launch YAML written to $CONFIG_FILE (source unchanged: $source_config)" } start_server() { @@ -1122,12 +1187,11 @@ start_server() { if [[ "$TRANSPORT" == "gpu_roce" ]]; then # Device-graph scheduler path: enqueue/get/reset run as DEVICE_CALLs # on the GPU and the captured RelayBP decode graph fires device-side. - # Provider selection and settings come only from YAML; - # dispatch-shape selection comes from the config's `dispatch: - # device_graph` key, injected at config-generation time. + # The temporary launch YAML prepared below makes the gpu_roce provider, + # device_graph dispatch, GPU, and runtime endpoint authoritative. # Eager module loading avoids lazy-load stalls inside the persistent # scheduler (same as the old bridge launcher). - prepare_gpu_roce_config "$peer_ip" "$remote_qp" + prepare_gpu_roce_config "$peer_ip" "$remote_qp" || return 1 CUDA_MODULE_LOADING=EAGER \ LD_LIBRARY_PATH="${server_ld_path}:${LD_LIBRARY_PATH:-}" \ "$SERVER_BIN" \ From 6f61b8aad5b18883988006ffb7b06910bd2d6198 Mon Sep 17 00:00:00 2001 From: Melody Ren Date: Wed, 12 Aug 2026 14:24:25 -0700 Subject: [PATCH 5/6] Align GPU RoCE page stride across FPGA path Signed-off-by: Melody Ren --- .../run_realtime_decoding.sh | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh index 29452c7ed..ab5c05076 100755 --- a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh +++ b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh @@ -939,7 +939,7 @@ setup_network_cpu_roce() { } prepare_gpu_roce_config() { - local peer_ip="$1" remote_qp="$2" num_pages="$3" + local peer_ip="$1" remote_qp="$2" num_pages="$3" page_size="$4" local remote_qp_decimal=$((remote_qp)) local payload_size=$((PAGE_SIZE - 24)) if (( payload_size <= 0 )); then @@ -973,7 +973,7 @@ transport: - --device=$BRIDGE_DEVICE - --peer-ip=$peer_ip - --remote-qp=$remote_qp_decimal - - --page-size=$PAGE_SIZE + - --page-size=$page_size - --num-pages=$num_pages - --payload-size=$payload_size ... @@ -982,14 +982,16 @@ EOF } start_roce_server() { - local peer_ip="$1" remote_qp="$2" server_log="$3" device_graph_num_pages="$4" + local peer_ip="$1" remote_qp="$2" server_log="$3" + local device_graph_num_pages="$4" device_graph_page_size="$5" _log "Starting decoding_server (wire=$WIRE, dispatch=$DISPATCH, remote-qp=$remote_qp)" local ready if [[ "$DISPATCH" == "device_graph" ]]; then # All-device_graph config: the standalone device-graph transceiver # brings up the GPU RoCE wire from the generated YAML; no transport # environment variables, CLI selector, or provider arguments are used. - prepare_gpu_roce_config "$peer_ip" "$remote_qp" "$device_graph_num_pages" || return 1 + prepare_gpu_roce_config "$peer_ip" "$remote_qp" \ + "$device_graph_num_pages" "$device_graph_page_size" || return 1 CUDA_MODULE_LOADING=EAGER \ "$SERVER_BIN" --config="$CONFIG_FILE" --timeout="$TIMEOUT" \ > >(tee "$server_log") 2>&1 & @@ -1029,6 +1031,9 @@ run_fpga() { # clamps), device_graph dispatch via DEVICE_GRAPH_NUM_PAGES here. local HSB_WQE_DEPTH=64 local DEVICE_GRAPH_NUM_PAGES="$HSB_WQE_DEPTH" + # GPU RoCE ring slots are 128-byte granular. Keep PAGE_SIZE as the frame + # budget, but give the provider and playback the same aligned slot stride. + local DEVICE_GRAPH_PAGE_SIZE=$(( ((PAGE_SIZE + 127) / 128) * 128 )) if (( NUM_SLOTS > HSB_WQE_DEPTH )); then _warn "NUM_SLOTS=$NUM_SLOTS exceeds the HSB WQE depth ($HSB_WQE_DEPTH); clamping" NUM_SLOTS="$HSB_WQE_DEPTH" @@ -1040,8 +1045,8 @@ run_fpga() { # configs) the server would reject the ring at startup, so fail fast # with the constraint spelled out. local host_page; host_page=$(getconf PAGESIZE) - if (( (DEVICE_GRAPH_NUM_PAGES * PAGE_SIZE) % host_page != 0 )); then - _err "device_graph ring ($DEVICE_GRAPH_NUM_PAGES slots x $PAGE_SIZE B) is not a multiple of this host's page size ($host_page B)." + if (( (DEVICE_GRAPH_NUM_PAGES * DEVICE_GRAPH_PAGE_SIZE) % host_page != 0 )); then + _err "device_graph ring ($DEVICE_GRAPH_NUM_PAGES slots x $DEVICE_GRAPH_PAGE_SIZE B) is not a multiple of this host's page size ($host_page B)." _err "The HSB frame stride must be a multiple of $(( host_page / HSB_WQE_DEPTH )) B on this host; see the unittests" _err "hsb_fpga_decoding_server_test.sh (--page-size) for a tunable-geometry run." exit 1 @@ -1050,7 +1055,8 @@ run_fpga() { : "${BRIDGE_DEVICE:=${IB_DEVICE:-rocep1s0f0}}" local server_log="$GEN_DIR/server.log" - start_roce_server "$FPGA_IP" "0x2" "$server_log" "$DEVICE_GRAPH_NUM_PAGES" || return 1 + start_roce_server "$FPGA_IP" "0x2" "$server_log" \ + "$DEVICE_GRAPH_NUM_PAGES" "$DEVICE_GRAPH_PAGE_SIZE" || return 1 _log "Streaming syndromes from the FPGA via playback (spacing=${SPACING}us)" # The FPGA writes syndrome frame rid to RDMA slot (rid % num-pages), so the @@ -1060,10 +1066,14 @@ run_fpga() { # (num_slots). The cpu_roce wire's server ring is NUM_SLOTS; the # device_graph ring is DEVICE_GRAPH_NUM_PAGES. local pb_pages="$NUM_SLOTS" - if [[ "$DISPATCH" == "device_graph" ]]; then pb_pages="$DEVICE_GRAPH_NUM_PAGES"; fi + local pb_page_size="$PAGE_SIZE" + if [[ "$DISPATCH" == "device_graph" ]]; then + pb_pages="$DEVICE_GRAPH_NUM_PAGES" + pb_page_size="$DEVICE_GRAPH_PAGE_SIZE" + fi local args=( --hsb-ip "$FPGA_IP" --per-round --config "$CONFIG_FILE" --syndromes "$SYNDROMES_FILE" --qp-number "$SERVER_QP" --rkey "$SERVER_RKEY" - --buffer-addr "$SERVER_ADDR" --page-size "$PAGE_SIZE" --num-pages "$pb_pages" ) + --buffer-addr "$SERVER_ADDR" --page-size "$pb_page_size" --num-pages "$pb_pages" ) $VERIFY && args+=(--verify) [[ -n "$NUM_SHOTS" ]] && args+=(--num-shots "$NUM_SHOTS") [[ -n "$SPACING" ]] && args+=(--spacing "$SPACING") From 31434646a0522cffad0c1ef2c10b8a30a032212d Mon Sep 17 00:00:00 2001 From: Melody Ren Date: Fri, 14 Aug 2026 22:13:35 -0700 Subject: [PATCH 6/6] [QEC] Parse HSB playback decoder config as YAML Signed-off-by: Melody Ren --- libs/qec/unittests/test_decoders_yaml.cpp | 43 +++++++ libs/qec/unittests/utils/CMakeLists.txt | 5 + .../utils/hsb_fpga_syndrome_playback.cpp | 120 +++++++++--------- 3 files changed, 108 insertions(+), 60 deletions(-) diff --git a/libs/qec/unittests/test_decoders_yaml.cpp b/libs/qec/unittests/test_decoders_yaml.cpp index bb1212dc1..c38551793 100644 --- a/libs/qec/unittests/test_decoders_yaml.cpp +++ b/libs/qec/unittests/test_decoders_yaml.cpp @@ -92,6 +92,49 @@ unexpected: true std::runtime_error); } +TEST(DecoderYAMLTest, AcceptsBlockStyleSparseMatrices) { + const std::string yaml = R"( +decoders: +- id: 0 + type: multi_error_lut + dispatch: device_graph + cuda_device_id: 0 + block_size: 2 + syndrome_size: 2 + H_sparse: + - 0 + - -1 + - 1 + - -1 + O_sparse: + - 1 + - -1 + D_sparse: + - 0 + - -1 + - 3 + - -1 +transport: + provider: gpu_roce + args: + - --device=mlx5_4 + - --peer-ip=192.168.0.2 +)"; + + const auto config = + cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str(yaml); + + ASSERT_EQ(config.decoders.size(), 1u); + const auto &decoder = config.decoders.front(); + EXPECT_EQ(decoder.H_sparse, (std::vector{0, -1, 1, -1})); + EXPECT_EQ(decoder.O_sparse, (std::vector{1, -1})); + EXPECT_EQ(decoder.D_sparse, (std::vector{0, -1, 3, -1})); + EXPECT_EQ(config.transport.provider, "gpu_roce"); + EXPECT_EQ( + config.transport.args, + (std::vector{"--device=mlx5_4", "--peer-ip=192.168.0.2"})); +} + /// Helper function to test that a decoder configuration can be serialized to /// and from YAML. void test_decoder_yaml_roundtrip( diff --git a/libs/qec/unittests/utils/CMakeLists.txt b/libs/qec/unittests/utils/CMakeLists.txt index cbfd7fdb1..4ffb65db5 100644 --- a/libs/qec/unittests/utils/CMakeLists.txt +++ b/libs/qec/unittests/utils/CMakeLists.txt @@ -195,11 +195,16 @@ target_include_directories(hsb_fpga_syndrome_playback target_link_libraries(hsb_fpga_syndrome_playback PRIVATE + cudaq-qec-realtime-decoding ${HSB_CORE_LIB} CUDA::cudart CUDA::cuda_driver Threads::Threads) +set_target_properties(hsb_fpga_syndrome_playback PROPERTIES + BUILD_RPATH "${CMAKE_BINARY_DIR}/lib" + INSTALL_RPATH "$ORIGIN/../lib") + # Ship the FPGA syndrome-playback tool as a deliverable alongside decoding_server # (see tools/decoding-server/CMakeLists.txt). This whole file is already gated by # the HSB tools, so this install is a no-op in the vanilla release and is diff --git a/libs/qec/unittests/utils/hsb_fpga_syndrome_playback.cpp b/libs/qec/unittests/utils/hsb_fpga_syndrome_playback.cpp index a0fcd2f09..aa759057e 100644 --- a/libs/qec/unittests/utils/hsb_fpga_syndrome_playback.cpp +++ b/libs/qec/unittests/utils/hsb_fpga_syndrome_playback.cpp @@ -6,6 +6,9 @@ * the terms of the Apache License 2.0 which accompanies this distribution. */ +#include "cudaq/qec/realtime/decoding_config.h" + +#include #include #include #include @@ -142,69 +145,23 @@ std::uint64_t parse_scalar(const std::string &content, } } -/// @brief Derive num_observables from O_sparse in the config. -/// -/// O_sparse encodes each observable as a row of correction indices terminated -/// by -1, so the number of row terminators is the number of observables. -std::size_t derive_num_observables(const std::string &content) { - std::size_t pos = content.find("O_sparse:"); - if (pos == std::string::npos) - return 0; - std::size_t bracket_start = content.find('[', pos); - if (bracket_start == std::string::npos) - return 0; - std::size_t bracket_end = content.find(']', bracket_start); - if (bracket_end == std::string::npos) - return 0; - std::string arr = - content.substr(bracket_start + 1, bracket_end - bracket_start - 1); - - std::size_t rows = 0; - std::istringstream ss(arr); - std::string token; - while (std::getline(ss, token, ',')) { - token.erase(0, token.find_first_not_of(" \t\n\r")); - token.erase(token.find_last_not_of(" \t\n\r") + 1); - if (token == "-1") - ++rows; - } - return rows; -} - /// @brief Derive num_measurements from D_sparse in the config. /// /// D_sparse encodes the detector-measurement matrix in row-major order with /// -1 as the row delimiter. The column indices are measurement indices, so /// max(column indices) + 1 = num_measurements. Returns 0 if D_sparse is /// absent or empty. -std::size_t derive_num_measurements(const std::string &content) { - std::size_t pos = content.find("D_sparse:"); - if (pos == std::string::npos) - return 0; - std::size_t bracket_start = content.find('[', pos); - if (bracket_start == std::string::npos) - return 0; - std::size_t bracket_end = content.find(']', bracket_start); - if (bracket_end == std::string::npos) - return 0; - std::string arr = - content.substr(bracket_start + 1, bracket_end - bracket_start - 1); - int max_col = -1; - std::istringstream ss(arr); - std::string token; - while (std::getline(ss, token, ',')) { - token.erase(0, token.find_first_not_of(" \t\n\r")); - token.erase(token.find_last_not_of(" \t\n\r") + 1); - if (token.empty()) +std::size_t derive_num_measurements(const std::vector &d_sparse) { + std::size_t span = 0; + for (const auto value : d_sparse) { + if (value < 0) continue; - try { - int val = std::stoi(token); - if (val >= 0 && val > max_col) - max_col = val; - } catch (...) { - } + const auto index = static_cast(value); + if (index >= std::numeric_limits::max()) + throw std::runtime_error("D_sparse measurement index is too large"); + span = std::max(span, static_cast(index + 1)); } - return (max_col >= 0) ? static_cast(max_col + 1) : 0; + return span; } // ============================================================================ @@ -1105,7 +1062,52 @@ int main(int argc, char **argv) { std::string config_content((std::istreambuf_iterator(config_file)), std::istreambuf_iterator()); - std::size_t syndrome_size = parse_scalar(config_content, "syndrome_size"); + std::size_t syndrome_size = 0; + std::size_t num_measurements = 0; + std::size_t num_observables = 0; + if (options.per_round) { + cudaq::qec::decoding::config::multi_decoder_config config; + try { + config = + cudaq::qec::decoding::config::multi_decoder_config::from_yaml_str( + config_content); + } catch (const std::exception &error) { + std::cerr << "Invalid config file " << config_path << ": " << error.what() + << "\n"; + return 1; + } + if (config.decoders.empty()) { + std::cerr << "Config file contains no decoders\n"; + return 1; + } + + const auto &decoder_config = config.decoders.front(); + if (decoder_config.syndrome_size > + std::numeric_limits::max()) { + std::cerr << "syndrome_size is too large\n"; + return 1; + } + syndrome_size = static_cast(decoder_config.syndrome_size); + try { + num_measurements = derive_num_measurements(decoder_config.D_sparse); + } catch (const std::exception &error) { + std::cerr << "Invalid config file " << config_path << ": " << error.what() + << "\n"; + return 1; + } + num_observables = static_cast(std::count( + decoder_config.O_sparse.begin(), decoder_config.O_sparse.end(), -1)); + } else { + // The predecoder playback path intentionally accepts a minimal YAML file + // containing only syndrome_size; it is not a decoder configuration. + const auto parsed_size = parse_scalar(config_content, "syndrome_size"); + if (parsed_size > std::numeric_limits::max()) { + std::cerr << "syndrome_size is too large\n"; + return 1; + } + syndrome_size = static_cast(parsed_size); + } + if (syndrome_size == 0) { std::cerr << "Invalid syndrome_size in config file\n"; return 1; @@ -1113,13 +1115,11 @@ int main(int argc, char **argv) { // num_measurements is the number of raw measurement bits per shot (used as // RPC payload size). Derived from D_sparse (max column index + 1). - // Falls back to syndrome_size for backward compat with configs that have - // no D matrix (e.g. mock decoder where measurements == syndromes). - std::size_t num_measurements = derive_num_measurements(config_content); + // Non-per-round predecoder configs have no D matrix because their input + // bits are already syndromes, so syndrome_size is their payload size. if (num_measurements == 0) num_measurements = syndrome_size; - const std::size_t num_observables = derive_num_observables(config_content); if (options.per_round && num_observables == 0) { std::cerr << "Per-round mode requires O_sparse in config file\n"; return 1;