diff --git a/libs/qec/lib/realtime/CMakeLists.txt b/libs/qec/lib/realtime/CMakeLists.txt index 8f9f45e1e..d6bd20adc 100644 --- a/libs/qec/lib/realtime/CMakeLists.txt +++ b/libs/qec/lib/realtime/CMakeLists.txt @@ -180,6 +180,7 @@ if(CMAKE_CUDA_COMPILER) COMPONENT qec-lib ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) + else() message(WARNING "cuda-quantum realtime headers not found. " "Device library cudaq-qec-realtime-device will not be built. " diff --git a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp b/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp index 1ce711dda..a02dbdd9f 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/DecodingSession.cpp @@ -7,18 +7,48 @@ ******************************************************************************/ #include "DecodingSession.h" +#include "DecodingServer.h" #include "RpcWireFormat.h" #include "../../hardware_guards.h" #include "cudaq/qec/logger.h" #include +#include #include #include +#include #include +#include #include namespace cudaq::qec::decoding_server { +namespace { + +std::optional env_int_optional(const char *name) { + const char *value = std::getenv(name); + if (!value || !*value) + return std::nullopt; + try { + return std::stoi(value); + } catch (const std::exception &) { + throw std::runtime_error(std::string("invalid ") + name + + " value: " + value); + } +} + +void set_graph_capture_device(const cudaq::qec::decoder &decoder) { + const int device = reconcile_gpu_roce_device( + env_int_optional("HOLOLINK_GPU_ID"), decoder.get_cuda_device_id()); + cudaq::qec::detail_affinity::set_cuda_device_for_decode(device); + if (device >= 0) + CUDA_QEC_INFO( + "DecodingSession::create: set CUDA device {} before graph capture", + device); +} + +} // namespace + // Busy high-water mark across all sessions (worker threads increment while // executing an item). static std::atomic g_busy_sessions{0}; @@ -53,6 +83,7 @@ DecodingSession::create(std::unique_ptr decoder, s->dec = std::move(decoder); if (s->dec->supports_graph_dispatch()) { + set_graph_capture_device(*s->dec); void *gr = s->dec->capture_decode_graph(); s->graph_resources = GraphResourcesPtr(gr, GraphResourcesDeleter{s->dec.get()}); diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceLinkCheck.cpp b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceLinkCheck.cpp index b209867ae..9fc37fe36 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceLinkCheck.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceLinkCheck.cpp @@ -20,9 +20,11 @@ struct ITransceiver; } extern "C" cudaq::qec::decoding_server::ITransceiver * -cudaqx_qec_make_gpu_roce_transceiver(); +cudaqx_qec_make_gpu_roce_transceiver(int pinned_cuda_device); -int main() { - (void)cudaqx_qec_make_gpu_roce_transceiver(); - return 0; -} +using GpuRoceFactoryFn = cudaq::qec::decoding_server::ITransceiver *(*)(int); + +static GpuRoceFactoryFn volatile gpu_roce_factory = + &cudaqx_qec_make_gpu_roce_transceiver; + +int main() { return gpu_roce_factory ? 0 : 1; } diff --git a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp index b1ee1f184..15e8f927e 100644 --- a/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp +++ b/libs/qec/lib/realtime/decoding-server-cqr/GpuRoceTransceiver.cpp @@ -19,9 +19,11 @@ #include #include #include +#include #include #include #include +#include // CUDAQ device-graph scheduler API (cudaq-realtime-dispatch). #include "cudaq/realtime/hololink_bridge_common.h" @@ -146,6 +148,23 @@ GpuRoceTransceiver::GpuRoceTransceiver(const GpuRoceConfig &config) size_t page_size = config.page_size ? config.page_size : config.frame_size; page_size = (page_size + 127) & ~static_cast(127); + if (page_size != 0 && + config.num_pages > std::numeric_limits::max() / page_size) + throw std::runtime_error( + "GpuRoceTransceiver: ring size overflow for " + "HOLOLINK_FRAME_SIZE/HOLOLINK_PAGE_SIZE=" + + std::to_string(page_size) + + " and HOLOLINK_NUM_PAGES=" + std::to_string(config.num_pages)); + const size_t ring_bytes = page_size * config.num_pages; + const long host_page_size = ::sysconf(_SC_PAGESIZE); + if (host_page_size > 0 && + ring_bytes % static_cast(host_page_size) != 0) + throw std::runtime_error( + "GpuRoceTransceiver: ring buffer size " + std::to_string(ring_bytes) + + " bytes is not aligned to host page size " + + std::to_string(host_page_size) + + " bytes; adjust HOLOLINK_NUM_PAGES or HOLOLINK_PAGE_SIZE"); + // Matches the call shape in hololink_qldpc_graph_decoder_bridge.cpp (lines // 288-291). transceiver_ = hololink_create_transceiver( diff --git a/libs/qec/tools/decoding-server/CMakeLists.txt b/libs/qec/tools/decoding-server/CMakeLists.txt index b001a0b08..a9c678dc7 100644 --- a/libs/qec/tools/decoding-server/CMakeLists.txt +++ b/libs/qec/tools/decoding-server/CMakeLists.txt @@ -97,6 +97,7 @@ if(CUDAQ_REALTIME_INCLUDE_DIR AND TARGET cudaq-qec-realtime-decoding-server-cqr) # presence and the proprietary archive are provisioned independently, so # treat a missing archive as "gpu_roce transport not available" and fall # back to the udp/cpu_roce transports rather than failing configuration. + set(_decoding_server_linked_proprietary_cudevice FALSE) if(CUDAQ_GPU_ROCE_AVAILABLE AND TARGET cudaq-qec-realtime-cudevice-proprietary) # Re-find in case the CQR subdir cache entry didn't propagate here. if(NOT CUDAQ_REALTIME_DISPATCH_LIBRARY) @@ -122,6 +123,7 @@ if(CUDAQ_REALTIME_INCLUDE_DIR AND TARGET cudaq-qec-realtime-decoding-server-cqr) ${CUDAQ_REALTIME_DISPATCH_LIBRARY} $ ) + set(_decoding_server_linked_proprietary_cudevice TRUE) target_link_options(decoding_server PRIVATE "LINKER:--export-dynamic" ) @@ -135,14 +137,25 @@ if(CUDAQ_REALTIME_INCLUDE_DIR AND TARGET cudaq-qec-realtime-decoding-server-cqr) endif() # RelayBP/nv-qldpc captures a CUDA device graph during decoder # initialization even when the server transport is udp or cpu_roce. Export - # the graph-dispatch shims whenever the cudevice archives are available so - # plugin dlsym(RTLD_DEFAULT, ...) lookups succeed. - if(TARGET cudaq-qec-realtime-cudevice AND TARGET cudaq-qec-realtime-cudevice-proprietary) + # the graph-dispatch shims whenever a cudevice archive is available so + # plugin dlsym(RTLD_DEFAULT, ...) lookups succeed. The proprietary archive + # also carries the public gpu_kernels.cu.o symbols, so do not whole-archive + # both the proprietary and public cudevice archives into the same binary. + if(TARGET cudaq-qec-realtime-cudevice OR TARGET cudaq-qec-realtime-cudevice-proprietary) target_link_libraries(decoding_server PRIVATE CUDA::cudart - $ - $ ) + if(TARGET cudaq-qec-realtime-cudevice-proprietary) + if(NOT _decoding_server_linked_proprietary_cudevice) + target_link_libraries(decoding_server PRIVATE + $ + ) + endif() + elseif(TARGET cudaq-qec-realtime-cudevice) + target_link_libraries(decoding_server PRIVATE + $ + ) + endif() target_link_options(decoding_server PRIVATE "LINKER:--export-dynamic" ) diff --git a/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp b/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp index b2890781b..8ac31a1d3 100644 --- a/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp +++ b/libs/qec/unittests/utils/hololink_fpga_syndrome_playback.cpp @@ -1399,37 +1399,29 @@ int main(int argc, char **argv) { if (options.verify) { std::cout << "\n=== ILA Capture & Verification ===\n"; - // In single-pass mode the player sends exactly num_shots packets, so the - // ILA buffer will not fill completely. Poll until the sample count - // stabilizes (no new samples for 2 consecutive checks). - constexpr int kStableChecks = 2; constexpr int kPollIntervalMs = 500; constexpr int kVerifyTimeoutMs = 30000; - std::cout << "Waiting for ILA capture to stabilize (timeout " - << kVerifyTimeoutMs << " ms)...\n"; + const std::uint32_t expected_samples = static_cast( + std::min(num_windows, ILA_DEPTH)); + std::cout << "Waiting for ILA capture to reach " << expected_samples + << " samples (timeout " << kVerifyTimeoutMs << " ms)...\n"; - std::uint32_t prev_count = 0; - int stable = 0; int elapsed = 0; while (elapsed < kVerifyTimeoutMs) { std::this_thread::sleep_for(std::chrono::milliseconds(kPollIntervalMs)); elapsed += kPollIntervalMs; - std::uint32_t count = ila_sample_count(*hololink); - if (count > 0 && count == prev_count) - ++stable; - else - stable = 0; - prev_count = count; - if (stable >= kStableChecks) + const std::uint32_t count = ila_sample_count(*hololink); + if (count >= expected_samples) break; } std::uint32_t actual_samples = ila_sample_count(*hololink); ila_disable(*hololink); - if (actual_samples == 0) { - std::cerr << "ILA: captured 0 samples (timeout " << kVerifyTimeoutMs - << " ms)\n"; + if (actual_samples < expected_samples) { + std::cerr << "ILA: captured " << actual_samples << " of " + << expected_samples << " expected samples (timeout " + << kVerifyTimeoutMs << " ms)\n"; return 1; } std::cout << "ILA: captured " << actual_samples << " samples\n"; diff --git a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh index 7b5c4945e..91d6b9ba9 100755 --- a/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh +++ b/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh @@ -105,8 +105,11 @@ GEN_SHOTS=100 # enqueue/get/reset run as DEVICE_CALLs on the GPU and # the captured RelayBP decode graph fires device-side) TRANSPORT="" -# GPU for the gpu_roce scheduler + decode graph (HOLOLINK_GPU_ID). +# GPU for the gpu_roce scheduler + decode graph. GPU_ID=0 +# Server-side GPU RoCE ring depth. "auto" chooses a page count whose total +# allocation satisfies the host page-size requirement. +GPU_ROCE_NUM_PAGES=auto # Runtime nv-qldpc plugin for the Relay BP profile: the prebuilt # libcudaq-qec-nv-qldpc-decoder.so, dlopen'd by both the generator (during @@ -129,10 +132,10 @@ MTU=4096 TIMEOUT=60 NUM_SHOTS="" PAGE_SIZE=384 -# Ring depth is intentionally NOT configurable: stock HSB posts WQE_NUM=64 -# receive/send WQEs, so a deeper ring aliases two slots per WQE and races -# RX/TX. The server clamps to 64 as well. +# CPU RoCE server ring slots. NUM_SLOTS=64 +# FPGA/emulator playback window pages. +PLAYBACK_NUM_PAGES=512 # TX SGE bytes for the server's SEND responses. RPCResponse (24B) + a # bit-packed correction byte fits well inside 64, keeping every response a # single 512-bit ILA beat. @@ -217,6 +220,9 @@ Run options: --frame-size N Server TX SGE bytes, cpu_roce only (default: 64; gpu_roce uses page-size as HOLOLINK_FRAME_SIZE) --gpu N GPU device id for gpu_roce (default: 0) + --gpu-roce-num-pages N Server GPU RoCE ring pages (default: auto-align; + starts from playback window pages) + --playback-num-pages N FPGA/emulator playback window pages (default: 512) --spacing N Inter-shot spacing in microseconds (default: 10) --control-port N UDP control port for emulator (default: 8193) @@ -235,6 +241,8 @@ while [[ $# -gt 0 ]]; do --onnx) ONNX_PATH="$2"; shift ;; --transport) TRANSPORT="$2"; shift ;; --gpu) GPU_ID="$2"; shift ;; + --gpu-roce-num-pages) GPU_ROCE_NUM_PAGES="$2"; shift ;; + --playback-num-pages) PLAYBACK_NUM_PAGES="$2"; shift ;; --nv-qldpc-plugin) NV_QLDPC_PLUGIN="$2"; shift ;; --config) CONFIG_FILE="$2"; shift ;; --syndromes) SYNDROMES_FILE="$2"; shift ;; @@ -280,6 +288,22 @@ if [[ "$TRANSPORT" != "cpu_roce" && "$TRANSPORT" != "gpu_roce" ]]; then exit 1 fi +# Some DOCA registrations require the gpu_roce server ring allocation to be +# host-page aligned. Keep playback capacity independent from the server ring, +# and choose a server page count that satisfies the allocation contract. +if [[ "$TRANSPORT" == "gpu_roce" && "$GPU_ROCE_NUM_PAGES" == "auto" ]]; then + HOST_PAGE_SIZE=$(getconf PAGESIZE 2>/dev/null || echo 4096) + SERVER_PAGE_SIZE=$(( ((PAGE_SIZE + 127) / 128) * 128 )) + GPU_ROCE_NUM_PAGES="$PLAYBACK_NUM_PAGES" + while (( (SERVER_PAGE_SIZE * GPU_ROCE_NUM_PAGES) % HOST_PAGE_SIZE != 0 )); do + ((GPU_ROCE_NUM_PAGES++)) + if (( GPU_ROCE_NUM_PAGES > 65536 )); then + echo "ERROR: unable to auto-align gpu_roce ring for page-size=$PAGE_SIZE host-page-size=$HOST_PAGE_SIZE" >&2 + exit 1 + fi + done +fi + # ============================================================================ # Logging Helpers # ============================================================================ @@ -798,18 +822,22 @@ generate_data_files() { fi # The server selects its transceiver from the per-decoder `transport:` YAML - # key (default cpu_roce). The generator doesn't emit non-default optional - # fields, so for the gpu_roce profile inject the key into our generated - # config, directly under the decoder's `type:` line. + # key (default cpu_roce). For gpu_roce, `cuda_device_id` pins graph capture + # and worker-thread execution to the selected GPU. The generator doesn't emit + # these non-default optional fields, so inject them into our generated config + # directly under the decoder's `type:` line. if [[ "$TRANSPORT" == "gpu_roce" ]]; then - _info "Injecting 'transport: gpu_roce' into $(basename "$CONFIG_FILE")" - awk '{ print } + _info "Injecting 'transport: gpu_roce' and cuda_device_id=$GPU_ID into $(basename "$CONFIG_FILE")" + awk -v gpu_id="$GPU_ID" '{ print } /^[[:space:]]*type:/ && !done { - print " transport: gpu_roce"; done = 1 + print " transport: gpu_roce" + print " cuda_device_id: " gpu_id + done = 1 }' "$CONFIG_FILE" > "${CONFIG_FILE}.tmp" \ && mv "${CONFIG_FILE}.tmp" "$CONFIG_FILE" - if ! grep -q "transport:.*gpu_roce" "$CONFIG_FILE"; then - _err "Failed to inject transport: gpu_roce into $CONFIG_FILE" + if ! grep -q "transport:.*gpu_roce" "$CONFIG_FILE" || \ + ! grep -q "cuda_device_id:.*$GPU_ID" "$CONFIG_FILE"; then + _err "Failed to inject gpu_roce transport/cuda_device_id into $CONFIG_FILE" return 1 fi fi @@ -980,7 +1008,7 @@ start_server() { HOLOLINK_PEER_IP="$peer_ip" \ HOLOLINK_REMOTE_QP="$((remote_qp))" \ HOLOLINK_FRAME_SIZE="$PAGE_SIZE" \ - HOLOLINK_NUM_PAGES="$NUM_SLOTS" \ + HOLOLINK_NUM_PAGES="$GPU_ROCE_NUM_PAGES" \ HOLOLINK_GPU_ID="$GPU_ID" \ "$SERVER_BIN" \ --config="$CONFIG_FILE" \ @@ -1057,6 +1085,7 @@ run_playback() { --rkey "$SERVER_RKEY" --buffer-addr "$SERVER_ADDR" --page-size "$PAGE_SIZE" + --num-pages "$PLAYBACK_NUM_PAGES" "$@" ) if $VERIFY; then @@ -1093,6 +1122,7 @@ run_emulated() { --port="$CONTROL_PORT" \ --bridge-ip="$BRIDGE_IP" \ --page-size="$PAGE_SIZE" \ + --num-pages="$PLAYBACK_NUM_PAGES" \ > >(tee "$emu_log") 2>&1 & local emu_pid=$! PIDS_TO_KILL+=("$emu_pid")