diff --git a/docker/decoding-server/dev.Dockerfile b/docker/decoding-server/dev.Dockerfile new file mode 100644 index 000000000..47e5bfa19 --- /dev/null +++ b/docker/decoding-server/dev.Dockerfile @@ -0,0 +1,249 @@ +# ============================================================================ # +# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # + +# Development image for building AND running the QEC decoding server on +# RDMA/FPGA hardware (NVQLink lab, DGX Spark / GB200). +# +# The base image (ghcr.io/nvidia/cudaqx-dev, built by +# docker/build_env/cudaqx.dev.Dockerfile) already carries the toolchain plus +# CUDA-Q and cudaq-realtime built at the .cudaq_version pin, with a copy of +# that pin at /cudaq_version. This file adds only the environment the +# hardware paths need -- the same packages CI installs at test time in +# .github/actions/build-lib/build_qec.sh, moved into cached image layers: +# +# - RDMA userspace (rdma-core, ibverbs providers incl. SoftRoCE/rxe) +# - DOCA 3.3.0 GPUNetIO dev headers (NOT doca-all: it conflicts with the +# Mellanox OFED preinstalled in the devcontainer base) +# - Holoscan SDK +# - TensorRT dev (for the trt_decoder plugin; arm64 packages exist for +# CUDA 13 only, hence the cu13.0 default base) +# - cuStabilizer (cuquantum-python wheel) +# - the Ising-artifact exporter's Python environment + `hf` CLI, so the +# hardware CI can download the gated model and rebuild the bundle on +# every run +# - a prebuilt holoscan-sensor-bridge 2.6.0-EA2 at /opt/holoscan-sensor-bridge +# +# What is deliberately NOT baked in: cudaq-realtime-with-HSB-tools and cudaqx +# itself -- the hardware CI builds both from the commit under test (see +# docker/decoding-server/hw_ci/container_build.sh). Proprietary artifacts +# (nv-qldpc plugin, cudevice archive) are bind-mounted at run time, never +# baked into a layer. +# +# Build (normally done by hw_ci/run_hw_ci.sh, context = this directory): +# docker build -f docker/decoding-server/dev.Dockerfile \ +# --build-arg base_image=ghcr.io/nvidia/cudaqx-dev: \ +# --build-arg cuda_native_arch=100 \ +# -t cudaqx-decoding-hwci docker/decoding-server + +ARG base_image=ghcr.io/nvidia/cudaqx-dev:latest-arm64-cu13.0 +FROM ${base_image} + +# CUDA architecture the prebaked holoscan-sensor-bridge kernels target: +# 100 = GB200 (sm_100), 121 = DGX Spark GB10 (sm_121). The hardware-CI +# runner auto-detects and passes this. +ARG cuda_native_arch=100 + +# --------------------------------------------------------------------------- +# Build tools + RDMA userspace. +# NOTE: the base image ships Mellanox OFED's rdma-core fork, whose +# ibverbs-providers outranks Ubuntu's and contains ONLY the mlx5 provider -- +# no rxe (SoftRoCE). The apt line below keeps the Mellanox package; the +# missing rxe provider is built from the matching Mellanox source in a later +# layer (see "SoftRoCE support" at the end of this file). perftest +# (ib_write_bw) is for fabric smoke tests. The `sudo` binary must exist +# because the example scripts' network helpers invoke it literally (a no-op +# when already root). +# --------------------------------------------------------------------------- +RUN apt-get update && apt-get install -y --no-install-recommends \ + ninja-build curl pkg-config jq \ + rdma-core ibverbs-providers ibverbs-utils infiniband-diags perftest \ + iproute2 ethtool iputils-ping sudo \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# --------------------------------------------------------------------------- +# DOCA 3.3.0: only the GPUNetIO dev package (mirrors build_qec.sh; doca-all +# conflicts with the base image's preinstalled OFED), plus cuda-nvrtc-dev +# matching the toolkit (hololink_core links CUDA::nvrtc). +# --------------------------------------------------------------------------- +RUN set -e; \ + DOCA_ARCH=$(uname -m); \ + case "$DOCA_ARCH" in aarch64|arm64) DOCA_ARCH="arm64-sbsa" ;; esac; \ + DOCA_REPO="https://linux.mellanox.com/public/repo/doca/3.3.0/ubuntu24.04/$DOCA_ARCH"; \ + curl -fsSL "$DOCA_REPO/GPG-KEY-Mellanox.pub" -o /usr/share/keyrings/GPG-KEY-Mellanox.pub; \ + echo "deb [signed-by=/usr/share/keyrings/GPG-KEY-Mellanox.pub] $DOCA_REPO /" \ + > /etc/apt/sources.list.d/doca.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends libdoca-sdk-gpunetio-dev; \ + CUDA_FULL_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p'); \ + CUDA_VER_DASH=$(echo "$CUDA_FULL_VERSION" | sed 's/\./-/'); \ + apt-get install -y cuda-nvrtc-dev-$CUDA_VER_DASH 2>/dev/null || true; \ + apt-get clean && rm -rf /var/lib/apt/lists/*; \ + test -d /opt/mellanox/doca/include + +# --------------------------------------------------------------------------- +# Holoscan SDK (force-install fallback mirrors build_qec.sh: the package's +# dependency list can miss on the devcontainer base). +# --------------------------------------------------------------------------- +RUN set -e; \ + CUDA_MAJOR_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p'); \ + apt-get update; \ + apt-get install -y --no-install-recommends holoscan-cuda-$CUDA_MAJOR_VERSION || { \ + _hsdk_tmp=$(mktemp -d); \ + (cd "$_hsdk_tmp" && apt-get download holoscan holoscan-cuda-$CUDA_MAJOR_VERSION \ + && dpkg --force-depends -i holoscan*.deb); \ + rm -rf "$_hsdk_tmp"; \ + }; \ + apt-get clean && rm -rf /var/lib/apt/lists/*; \ + test -d /opt/nvidia/holoscan + +# --------------------------------------------------------------------------- +# TensorRT dev, pinned to the toolkit's CUDA flavor (mirrors the arm64 steps +# in .github/workflows/lib_qec.yaml). +# --------------------------------------------------------------------------- +RUN set -e; \ + CUDA_FULL_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p'); \ + apt-get update; \ + apt-cache search tensorrt \ + | awk -v v="$CUDA_FULL_VERSION" '{print "Package: "$1"\nPin: version *+cuda"v"\nPin-Priority: 1001\n"}' \ + > /etc/apt/preferences.d/tensorrt-cuda$CUDA_FULL_VERSION.pref; \ + apt-get install -y tensorrt-dev; \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# --------------------------------------------------------------------------- +# cuStabilizer (pre-bakes .github/actions/build-lib/setup_custabilizer.sh's +# pip install; sourcing that script at build time then becomes a no-op). +# --------------------------------------------------------------------------- +RUN set -e; \ + CUDA_MAJOR_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p'); \ + pip install --no-cache-dir "cuquantum-python-cu${CUDA_MAJOR_VERSION}>=26.3.0" + +# --------------------------------------------------------------------------- +# Ising exporter environment: the `hf` CLI plus every Python package +# examples/qec/realtime_decoding_demo/prepare_ising_artifacts.py checks for, +# so the hardware CI downloads the gated model and regenerates the bundle on +# every run. Torch must be a CUDA build: the pinned Ising-Decoding exporter +# is GPU-only (its local_run.sh preflights torch.cuda.is_available(), so CPU +# torch fails the lane before inference starts). The cu130 index matches +# the image's toolkit and covers GB200 (sm_100) and Spark GB10 (sm_121); +# the multi-GB nvidia-* dependency wheels are the accepted cost. +# --------------------------------------------------------------------------- +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu130 \ + && pip install --no-cache-dir \ + "huggingface_hub[cli]" \ + stim ldpc beliefmatching hydra-core omegaconf onnx \ + pymatching safetensors scipy matplotlib numpy \ + && hf version + +# --------------------------------------------------------------------------- +# Prebuilt holoscan-sensor-bridge 2.6.0-EA2 (mirrors build_qec.sh: same +# operator strip, same targets). Both the source and build trees are kept: +# cudaq-realtime's HSB-tools build and cudaqx's HSB-tools build consume them +# via HOLOSCAN_SENSOR_BRIDGE_{SOURCE,BUILD}_DIR. The /workspaces symlink +# satisfies hsb_fpga_decoding_server_test.sh's default HSB_DIR. +# --------------------------------------------------------------------------- +RUN set -e; \ + git clone --depth 1 --branch 2.6.0-EA2 \ + https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git \ + /opt/holoscan-sensor-bridge; \ + cd /opt/holoscan-sensor-bridge; \ + sed -i '/add_subdirectory(audio_packetizer)/d; /add_subdirectory(compute_crc)/d; \ + /add_subdirectory(csi_to_bayer)/d; /add_subdirectory(image_processor)/d; \ + /add_subdirectory(iq_dec)/d; /add_subdirectory(iq_enc)/d; \ + /add_subdirectory(linux_coe_receiver)/d; /add_subdirectory(linux_receiver)/d; \ + /add_subdirectory(packed_format_converter)/d; /add_subdirectory(sub_frame_combiner)/d; \ + /add_subdirectory(udp_transmitter)/d; /add_subdirectory(emulator)/d; \ + /add_subdirectory(sig_gen)/d; /add_subdirectory(sig_viewer)/d' \ + src/hololink/operators/CMakeLists.txt; \ + export CUDA_NATIVE_ARCH=${cuda_native_arch}; \ + cmake -G Ninja -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DHOLOLINK_BUILD_ONLY_NATIVE=OFF \ + -DHOLOLINK_BUILD_PYTHON=OFF \ + -DHOLOLINK_BUILD_TESTS=OFF \ + -DHOLOLINK_BUILD_TOOLS=OFF \ + -DHOLOLINK_BUILD_EXAMPLES=OFF \ + -DHOLOLINK_BUILD_EMULATOR=OFF; \ + cmake --build build --target gpu_roce_transceiver hololink_core; \ + mkdir -p /workspaces; \ + ln -sfn /opt/holoscan-sensor-bridge /workspaces/holoscan-sensor-bridge + +# --------------------------------------------------------------------------- +# Tools the base image lacks: +# - ibdev2netdev: the in-tree --setup-network helpers shell out to it to map +# IB devices to netdevs. It lives in mlnx-ofed-kernel-utils (from the +# DOCA repo configured above) -- NOT in modern mlnx-tools, which dropped +# it. Userspace deps only; no DKMS/kernel modules ride along with +# --no-install-recommends. +# - patchelf: libs/qec's add_target_libs_to_wheel patches the rpath of +# staged external decoder plugins at configure time. +# --------------------------------------------------------------------------- +RUN apt-get update && apt-get install -y --no-install-recommends \ + mlnx-ofed-kernel-utils patchelf \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && test -x /usr/sbin/ibdev2netdev && command -v patchelf + +# --------------------------------------------------------------------------- +# SoftRoCE support, part 1: the rxe userspace provider. The Mellanox +# rdma-core fork installed above ships only the mlx5 provider, and its +# provider ABI (rdmav59) differs from Ubuntu's rdma-core, so the Ubuntu +# ibverbs-providers package cannot supply librxe either. Build it from the +# SAME Mellanox rdma-core source release (the DOCA SOURCES bundle; the rxe +# provider is shipped there but `if (0)`-disabled in CMakeLists.txt) and +# install just the provider .so. The dpkg-query / PABI derivations keep +# this layer loudly consistent with whatever rdma-core version the DOCA +# repo actually installed. kmod supplies insmod for part 2. +# --------------------------------------------------------------------------- +ARG mlnx_ofed_src_ver=26.01-1.0.0.0 +ARG mlnx_ofed_src_sha256=ed5597a547c2d5bb858b43f2305ec19f539bc70c4e5ed75aa6c6897a715568d3 +# NOTE: no apt cmake here -- the base image ships cmake 4.x under +# /usr/local/cmake-*/bin, and noble's cmake 3.28 would shadow it at +# /bin/cmake, breaking the later in-container source build (cudaq-realtime +# requires cmake >= 4.0). rdma-core needs >= 3.18.1, satisfied by the base. +RUN set -e; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ninja-build pkg-config patch kmod \ + libnl-3-dev libnl-route-3-dev libudev-dev; \ + RDMA_CORE_VER=$(dpkg-query -W -f '${Version}' rdma-core); \ + RDMA_CORE_VER=${RDMA_CORE_VER%-*}; \ + MULTIARCH=$(gcc -print-multiarch); \ + PABI=$(basename /usr/lib/$MULTIARCH/libibverbs/libmlx5-rdmav*.so \ + | sed 's|libmlx5-rdmav\([0-9]*\)\.so|\1|'); \ + tmp=$(mktemp -d); cd "$tmp"; \ + curl -fsSLO "https://linux.mellanox.com/public/repo/doca/3.3.0/SOURCES/mlnx_ofed/MLNX_OFED_SRC-debian-${mlnx_ofed_src_ver}.tgz"; \ + echo "${mlnx_ofed_src_sha256} MLNX_OFED_SRC-debian-${mlnx_ofed_src_ver}.tgz" | sha256sum -c -; \ + tar xzf "MLNX_OFED_SRC-debian-${mlnx_ofed_src_ver}.tgz" \ + "MLNX_OFED_SRC-${mlnx_ofed_src_ver}/SOURCES/rdma-core_${RDMA_CORE_VER}.orig.tar.gz"; \ + tar xzf "MLNX_OFED_SRC-${mlnx_ofed_src_ver}/SOURCES/rdma-core_${RDMA_CORE_VER}.orig.tar.gz"; \ + cd "rdma-core-${RDMA_CORE_VER}"; \ + sed -i 's|^add_subdirectory(providers/mlx5)$|add_subdirectory(providers/mlx5)\nadd_subdirectory(providers/rxe)|' \ + CMakeLists.txt; \ + cmake -GNinja -S . -B build -DNO_MAN_PAGES=1 -DNO_PYVERBS=1 >/dev/null; \ + ninja -C build "librxe-rdmav${PABI}.so"; \ + install -m 644 "build/lib/librxe-rdmav${PABI}.so" "/usr/lib/$MULTIARCH/libibverbs/"; \ + printf 'driver rxe\n' > /etc/libibverbs.d/rxe.driver; \ + cd /; rm -rf "$tmp"; \ + apt-get clean && rm -rf /var/lib/apt/lists/*; \ + test -f "/usr/lib/$MULTIARCH/libibverbs/librxe-rdmav${PABI}.so" + +# --------------------------------------------------------------------------- +# SoftRoCE support, part 2: OFED-compat rdma_rxe kernel module SOURCE (see +# hw_ci/rxe-ofed/README.md). On hosts whose ib_core comes from DOCA/MLNX- +# OFED DKMS (e.g. GB200 #2), the distro's in-tree rdma_rxe.ko cannot load +# (symbol CRC mismatch), so run_hw_ci.sh's rxe mode builds this patched +# copy of the upstream rxe driver against the host's kernel headers + +# ofa_kernel tree (mounted at /lib/modules and /usr/src) inside the +# privileged container and insmods it. The image only STAGES the patched +# source; the compile is per-host at container setup time. +# --------------------------------------------------------------------------- +ARG rxe_kernel_ref=v6.17 +COPY hw_ci/rxe-ofed /opt/rxe-ofed +RUN /opt/rxe-ofed/prepare-src.sh "${rxe_kernel_ref}" /opt/rxe-ofed/src + +ENV HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=/opt/holoscan-sensor-bridge \ + HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=/opt/holoscan-sensor-bridge/build diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md new file mode 100644 index 000000000..530b9a5ae --- /dev/null +++ b/docker/decoding-server/hw_ci/README.md @@ -0,0 +1,141 @@ +# Decoding-server hardware CI (NVQLink lab) + +CI-like testing of the QEC decoding server on real hardware — ConnectX RDMA +NICs, an FPGA syndrome source, and CC >= 9.0 GPUs — none of which normal +GitHub CI has (its GPU runners are A100/L4, so every `device_graph` / +CUDA-graph path is unreachable there). + +`run_hw_ci.sh` takes a cudaqx commit, clones it, builds the dev image locally +(`../dev.Dockerfile`, layer-cached), builds cudaq-realtime + cudaqx + the +`realtime_decoding_demo` binaries inside the container +(`container_build.sh`), and runs the test lanes. Every lane ends the summary +as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. + +``` +./run_hw_ci.sh --sha --hf-token-prompt +./run_hw_ci.sh --list # show the lane set +``` + +The commit is cloned from the repo the script itself lives in, so local +(unpushed) commits are testable directly. SoftRoCE (`--roce-pair rxe`) and +the GB200 lab FPGA port (`--fpga-device mlx5_4`) are the defaults. + +## Lanes + +* **examples tier** — the shipped `examples/qec/realtime_decoding_demo` + driver: all 4 decoders (pymatching, multi_error_lut, nv-qldpc-decoder, + and the TensorRT plugin running the exported Ising model — lane name + `trt_decoder(ising)`) over `udp` (baseline), `cpu_roce` two-process (RoCE pair), + and the FPGA source (`cpu_roce` host dispatch ×4, `device_graph` + nv-qldpc ×1); plus `ising-prepare`, which downloads the gated Hugging Face + Ising model and rebuilds the TRT artifact bundle **on every run** — a FAIL + there means the HF/download/export path regressed, independent of decoders. +* **extra tier** — hardware tests outside the examples directory: + `DecodingServerTwoProcess` + the `surface_code-1-cqr` two-process app + ctests over `cpu_roce`; `hsb_fpga_decoding_server_test.sh` over `cpu_roce` + and `gpu_roce` (the FPGA control-plane / SIF playback coverage); the + CC >= 9.0-only ctests (`test_realtime_qldpc_graph_decoding`, + mixed-dispatch, FP8 ONNX); and the `gpu_roce` QLDPC bridge as a + below-the-server cross-check. +* **opt-in** (`--include-opt-in`) — the `gpu_roce` predecoder bridge; it + links the experimental `cudaq-realtime-pipeline` library (off by default, + pending a port to the post-PR4770 graph-launch API), so today it SKIPs. + +## One-time host setup + +1. Docker + nvidia-container-toolkit; user in the `docker` group; + `docker login ghcr.io` (to pull `ghcr.io/nvidia/cudaqx-dev`). +2. For SoftRoCE mode (`--roce-pair rxe`): `sudo modprobe rdma_rxe`, persisted + via `echo rdma_rxe | sudo tee /etc/modules-load.d/rdma_rxe.conf`. + On hosts whose `ib_core` comes from DOCA/MLNX-OFED DKMS (e.g. a GB200 + with `doca-ofed` installed), the distro module refuses to load + (`disagrees about version of symbol ib_*`); the runner then builds an + OFED-compat `rdma_rxe` from the source staged in the image and loads it + from the privileged container automatically — no host setup needed, but + the module does not persist across host reboots (see + `rxe-ofed/README.md`). The image also builds the rxe userspace provider + that Mellanox's `ibverbs-providers` omits, from the matching Mellanox + rdma-core source release, so `ibv_devinfo` can open rxe devices + in-container. +3. FPGA cabled/flashed and reachable (defaults: NIC 192.168.0.1/24, FPGA + 192.168.0.2). Machines whose single cable is wired as a loopback pair + instead run with `--no-fpga --roce-pair DEV0,DEV1`. +4. Proprietary artifacts (any subset; missing pieces => named SKIPs), staged + under `--artifacts-dir` (default `/opt/nvqlink-lab-artifacts`), mounted + read-only at `/artifacts` in the container: + + ``` + decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so # build against the SHA under test + cudevice/libcudaq-qec-realtime-cudevice-proprietary.a + ising-bundle/ # optional: pre-built Ising/TRT + # bundle for tokenless machines + ``` + +5. Hugging Face access for the Ising lanes: request access once to the gated + `nvidia/Ising-Decoder-SurfaceCode-1-Fast` repo, then provide a token via + `--hf-token-file`, `HF_TOKEN`, or — on shared/public accounts where the + token must not touch disk — `--hf-token-prompt` (interactive, hidden + input). The token is passed only to the `ising-prepare` lane's + `docker exec` — never baked into an image or written to a log. + Without a token the trt lanes fall back to a pre-built bundle at + `/ising-bundle/` when one is staged (generate it once on + a token-holding machine with `prepare_ising_artifacts.py prepare + --app --artifacts-dir ` and copy the + directory over); `ising-prepare` then SKIPs with a reason noting the HF + download/export path was not exercised. No token and no staged bundle + => the ising/trt lanes SKIP. +6. ~60 GB free disk for image layers and build trees. +7. **One run per machine at a time.** The FPGA (SIF/BRAM/ILA), the + SoftRoCE objects, and the RoCE pair addressing are host-global, so the + runner serializes itself with a host-wide lock + (`/tmp/cudaqx-hw-ci.lock`). A second invocation fails immediately, + naming the active run's user/pid/sha; `--lock-wait` queues behind it + instead. A crashed or killed run releases the lock automatically (the + kernel drops it with the process — the lock file itself is inert), so + only a live-but-wedged run can hold others out: `--force-unlock` kills + whatever actually holds the lock and proceeds (for another user's + wedged run: `sudo fuser -vk /tmp/cudaqx-hw-ci.lock`). +8. NetworkManager-managed ports (the DGX OS default) silently drop the + runner's statically assigned `10.0.0.x` pair addresses on NM's DHCP + retry timer. The runner re-asserts the addresses and waits for the + IPv4-mapped RoCE GIDs before every cpu_roce lane, which is normally + enough; the permanent fix is to mark the pair unmanaged: + `sudo nmcli device set managed no` (once per port). + +## Per-machine invocations + +```bash +# GB200 (lab default wiring: FPGA on mlx5_4, SoftRoCE for two-process; +# artifacts staged in the home directory rather than the /opt default): +./run_hw_ci.sh --sha --hf-token-prompt \ + --artifacts-dir ~/nvqlink-lab-artifacts + +# DGX Spark, single cable in loopback mode (port0 <-> port1, no FPGA): +./run_hw_ci.sh --sha --no-fpga --roce-pair rocep1s0f0,rocep1s0f1 + +# DGX Spark, single cable in FPGA mode (use the port actually cabled to the +# FPGA -- on the current lab Spark that is roceP2p1s0f0; a wrong-but-linked +# port fails with "ILA: captured 0 of N expected samples"): +./run_hw_ci.sh --sha --fpga-device roceP2p1s0f0 +``` +If that clone is owned by another user — e.g. it was created from a +devcontainer running as root — git refuses to serve it ("dubious +ownership"); allow it once with +`git config --global --add safe.directory ` and the same for +`/.git` (the clone accesses the repo by its `.git` path). + +The CUDA architecture is auto-detected (`--cuda-arch` to override; GB200 = +100, Spark GB10 = 121). On 64 KiB-page kernels (GB200 `-64k`) the ring slot +size defaults to 512 and the `device_graph` lane rounds up to the +host-page-compatible value; `--page-size` overrides. + +## Image / pin lifecycle + +The dev image is built locally on every run and never pushed; the Docker +layer cache makes unchanged builds take seconds. Its base +(`ghcr.io/nvidia/cudaqx-dev`) carries CUDA-Q at the `.cudaq_version` pin — +the runner picks the pin-matched base tag for the commit under test and +fails fast on a mismatch. For pre-merge commits that bump `.cudaq_version` +(no published base yet), `--build-base` builds the base locally from the +commit's own `docker/build_env/cudaqx.dev.Dockerfile` (multi-hour the first +time per pin, cached after). `--refresh-base` re-pulls a moved base tag. diff --git a/docker/decoding-server/hw_ci/container_build.sh b/docker/decoding-server/hw_ci/container_build.sh new file mode 100755 index 000000000..2b6e95016 --- /dev/null +++ b/docker/decoding-server/hw_ci/container_build.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# ============================================================================ # +# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# In-container build for the decoding-server hardware CI. Runs inside the +# dev image (docker/decoding-server/dev.Dockerfile) with the cudaqx checkout +# under test mounted at /workspaces/cudaqx and optional proprietary artifacts +# mounted read-only at /artifacts. +# +# This mirrors the CI recipe in .github/actions/build-lib/build_qec.sh with +# the apt/DOCA/Holoscan/HSB steps removed (baked into the image) and the +# hardware-lab deltas applied: +# - CUDA architecture comes from the runner (GB200=100, DGX Spark=121) +# - holoscan-sensor-bridge is the image's prebuilt /opt tree +# - the proprietary cudevice archive and nv-qldpc plugin are picked up +# from /artifacts when present (never required: absent pieces surface +# later as per-lane SKIPs, not build failures) +# - the trt_decoder plugin build is forced ON (TensorRT is in the image; +# a detection regression should fail the configure loudly) +# - the realtime_decoding_demo example binaries are built as well +# +# Usage (normally invoked by run_hw_ci.sh via docker exec): +# container_build.sh --cuda-arch N +set -euo pipefail + +CUDA_ARCH="" +while [[ $# -gt 0 ]]; do + case "$1" in + --cuda-arch) CUDA_ARCH="$2"; shift ;; + *) echo "ERROR: unknown option: $1" >&2; exit 1 ;; + esac + shift +done +[[ -n "$CUDA_ARCH" ]] || { echo "ERROR: --cuda-arch is required" >&2; exit 1; } + +CUDAQX_SRC=/workspaces/cudaqx +CUDAQ_PREFIX=${CUDAQ_INSTALL_PREFIX:-/usr/local/cudaq} +CUDAQX_INSTALL_PREFIX=/usr/local/cudaqx +CUDAQ_REALTIME_ROOT=/tmp/cudaq-realtime +HSB_ROOT=${HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR:-/opt/holoscan-sensor-bridge} +HSB_BUILD=${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR:-$HSB_ROOT/build} +ARTIFACTS_DIR=/artifacts +NV_QLDPC_PLUGIN=$ARTIFACTS_DIR/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so +CUDEVICE_ARCHIVE=$ARTIFACTS_DIR/cudevice/libcudaq-qec-realtime-cudevice-proprietary.a + +cd "$CUDAQX_SRC" +# setup_custabilizer.sh expands $CUSTABILIZER_ROOT unguarded, which is fatal +# under this script's `set -u` (the CI caller runs without -u). +export CUSTABILIZER_ROOT="${CUSTABILIZER_ROOT:-}" +. .github/actions/build-lib/setup_custabilizer.sh # no-op: wheel is baked in +. scripts/cudaq_realtime_cmake_flags.sh # AVX512 workaround, x86-only + +export CUDA_NATIVE_ARCH="$CUDA_ARCH" + +# --------------------------------------------------------------------------- +# cudaq-realtime with HSB tools, from the commit's own .cudaq_version pin +# (produces libcudaq-realtime-bridge-gpu-roce.so, the provider decoding_server +# dlopens for the gpu_roce wire). Mirrors build_qec.sh lines 12-115. +# --------------------------------------------------------------------------- +CUDAQ_REPO=${CUDAQ_REPO:-$(jq -r '.cudaq.repository' .cudaq_version)} +CUDAQ_REF=${CUDAQ_REF:-$(jq -r '.cudaq.ref' .cudaq_version)} +echo "== cudaq-realtime source: ${CUDAQ_REPO}@${CUDAQ_REF}" + +cd /tmp +rm -rf cudaq-realtime-src "$CUDAQ_REALTIME_ROOT" +git clone --filter=blob:none --no-checkout "https://github.com/${CUDAQ_REPO}.git" cudaq-realtime-src +cd cudaq-realtime-src +git sparse-checkout init --cone +git sparse-checkout set realtime cmake +git checkout "$CUDAQ_REF" + +cd realtime +mkdir -p build && cd build +cmake -G Ninja -DCMAKE_INSTALL_PREFIX="$CUDAQ_REALTIME_ROOT" \ + -DCMAKE_CUDA_FLAGS="$(cudaq_realtime_cmake_cuda_flags)" \ + -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCH" \ + -DCUDAQ_REALTIME_ENABLE_HSB_TOOLS=ON \ + -DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR="$HSB_ROOT" \ + -DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR="$HSB_BUILD" \ + .. +ninja +ninja install + +# --------------------------------------------------------------------------- +# Proprietary inputs (optional). +# --------------------------------------------------------------------------- +_prop_archive_flag="" +if [[ -f "$CUDEVICE_ARCHIVE" ]]; then + _prop_archive_flag="-DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=$CUDEVICE_ARCHIVE" + echo "== cudevice proprietary archive: $CUDEVICE_ARCHIVE" +else + echo "== cudevice proprietary archive: absent (device_graph-dispatch tests will SKIP)" +fi +if [[ -f "$NV_QLDPC_PLUGIN" ]]; then + # cudaqx's external-decoder install patches the plugin's rpath IN PLACE + # and /artifacts is mounted read-only -- hand the build a writable copy + # (the decoder-plugins symlink below then serves the patched copy too). + mkdir -p /tmp/hwci-artifacts/decoder-plugins + cp -f "$NV_QLDPC_PLUGIN" /tmp/hwci-artifacts/decoder-plugins/ + NV_QLDPC_PLUGIN="/tmp/hwci-artifacts/decoder-plugins/$(basename "$NV_QLDPC_PLUGIN")" + # Configure-time gate for test_realtime_qldpc_graph_decoding and the + # mixed-dispatch app example. + export QEC_EXTERNAL_DECODERS="$NV_QLDPC_PLUGIN" + echo "== nv-qldpc plugin: $NV_QLDPC_PLUGIN (writable copy from $ARTIFACTS_DIR)" +else + echo "== nv-qldpc plugin: absent (nv-qldpc lanes will SKIP)" +fi + +# --------------------------------------------------------------------------- +# cudaqx qec. Configured from the TOP-LEVEL CMakeLists (not -S libs/qec) so +# the build tree lands at build/libs/qec/... -- the layout every in-tree +# hardware script hardcodes (hsb_fpga_decoding_server_test.sh & co. default +# to CUDAQX_DIR=/workspaces/cudaqx and resolve binaries + LD paths from +# build/libs/qec/...). Mirrors build_qec.sh lines 120-141 plus the deltas +# listed in the header. +# --------------------------------------------------------------------------- +cd "$CUDAQX_SRC" +cmake -S . -B build \ + -DCUDAQX_ENABLE_LIBS=qec \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc-12 \ + -DCMAKE_CXX_COMPILER=g++-12 \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCH" \ + -DCUDAQ_DIR="$CUDAQ_PREFIX/lib/cmake/cudaq/" \ + -DCUDAQX_INCLUDE_TESTS=ON \ + -DCUDAQX_BINDINGS_PYTHON=ON \ + -DCMAKE_INSTALL_PREFIX="$CUDAQX_INSTALL_PREFIX" \ + -DCUDAQ_REALTIME_ROOT="$CUDAQ_REALTIME_ROOT" \ + -DCUDAQX_QEC_ENABLE_HSB_TOOLS=ON \ + -DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR="$HSB_ROOT" \ + -DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR="$HSB_BUILD" \ + -DCUDAQ_QEC_BUILD_TRT_DECODER=ON \ + $_prop_archive_flag +cmake --build build --target install -j "$(nproc)" + +# The decoding server discovers decoder plugins in +# /lib/decoder-plugins; expose the proprietary one when staged. +if [[ -f "$NV_QLDPC_PLUGIN" ]]; then + mkdir -p "$CUDAQX_INSTALL_PREFIX/lib/decoder-plugins" + ln -sfn "$NV_QLDPC_PLUGIN" \ + "$CUDAQX_INSTALL_PREFIX/lib/decoder-plugins/$(basename "$NV_QLDPC_PLUGIN")" +fi + +# --------------------------------------------------------------------------- +# realtime_decoding_demo example binaries (the examples-tier lanes). The +# arch override is mandatory: the demo's CMakeLists defaults to 80. +# --------------------------------------------------------------------------- +cmake -S docs/sphinx/examples/qec/realtime_decoding_demo -B demo-build -G Ninja \ + -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCH" \ + -DCUDAQ_INSTALL_DIR="$CUDAQ_PREFIX" \ + -DCUDAQX_INSTALL_DIR="$CUDAQX_INSTALL_PREFIX" \ + -DCUDAQ_REALTIME_DIR="$CUDAQ_REALTIME_ROOT" +cmake --build demo-build -j "$(nproc)" + +echo "== container build complete" +echo " cudaqx install : $CUDAQX_INSTALL_PREFIX" +echo " realtime : $CUDAQ_REALTIME_ROOT" +echo " qec build tree : $CUDAQX_SRC/build" +echo " demo binaries : $CUDAQX_SRC/demo-build" diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh new file mode 100755 index 000000000..efae15453 --- /dev/null +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -0,0 +1,867 @@ +#!/bin/bash +# ============================================================================ # +# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# Hardware CI runner for the QEC decoding server (NVQLink lab: DGX Spark / +# GB200 with ConnectX NICs and an FPGA syndrome source). +# +# Given a cudaqx commit, this script -- run on the HOST, not in a container: +# 1. clones cudaqx at that commit into the work dir +# 2. builds the dev image (docker/decoding-server/dev.Dockerfile) locally; +# the Docker layer cache makes unchanged builds near-instant +# 3. fails fast if the image's baked CUDA-Q does not match the commit's +# .cudaq_version pin +# 4. builds cudaq-realtime + cudaqx + the demo binaries inside the +# container (hw_ci/container_build.sh) +# 5. runs the test lanes (see --list) and prints a PASS/FAIL/SKIP summary +# +# Lanes report SKIP (exit 77) when an input is absent -- a proprietary +# artifact, the HF token, the FPGA -- so lost coverage is always visible in +# the summary without failing the run; --strict turns skips into failures. +# +# Proprietary artifacts are staged once on the host (see --artifacts-dir); +# the Ising/TRT model bundle is deliberately NOT staged: the ising-prepare +# lane downloads the gated Hugging Face model and rebuilds the bundle inside +# the container on every run, so that path is continuously validated. +set -uo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +SHA="" +# The repo under test is the one this script lives in; a fresh clone of it +# is made at the requested --sha. +REPO_URL="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel 2>/dev/null)" +[[ -n "$REPO_URL" ]] || { echo "error: cannot resolve the containing git repo (is the script inside a clone?)" >&2; exit 1; } +WORKDIR="$HOME/.cache/cudaqx-hw-ci" +ARTIFACTS_DIR="/opt/nvqlink-lab-artifacts" +TIER="all" # examples | extra | all +INCLUDE_OPT_IN=false +ONLY_GLOB="" +SKIP_GLOB="" +STRICT=false +LIST_ONLY=false +REFRESH_BASE=false +BASE_IMAGE="" # resolved from the pin unless given +BUILD_BASE=false +CUDA_VERSION="13.0" +CUDA_ARCH="" # auto-detected unless given (Spark=121, GB200=100) +ROCE_PAIR="rxe" # rxe | DEV0,DEV1 | (empty via --roce-pair none = skip cpu_roce pair lanes) +HF_TOKEN_FILE="" +HF_TOKEN_PROMPT=false +FPGA_DEVICE="mlx5_4" # ConnectX IB device facing the FPGA (GB200 lab wiring) +BRIDGE_IP="192.168.0.1" +FPGA_IP="192.168.0.2" +PAGE_SIZE="" # default derived from the host page size +KEEP_CONTAINER=false +NO_FPGA=false +LOCK_WAIT=false # queue behind an active run instead of failing +FORCE_UNLOCK=false # kill a wedged lock holder, then run + +print_usage() { + cat < the dependent lanes SKIP + +Lane selection: + --tier T examples | extra | all (default: all) + --include-opt-in also run the opt-in lanes (see --list) + --only GLOB run only lanes matching GLOB (e.g. 'examples/fpga/*') + --skip GLOB skip lanes matching GLOB + --list print the lane list for the current flags and exit + --strict any SKIP fails the run (full-coverage mode) + +Image: + --base-image IMG override the cudaqx-dev base image + --refresh-base docker build --pull (accept a moved base tag; + invalidates cached layers built on the old base) + --build-base if the pin-matched base tag cannot be pulled, build + it locally from the commit's cudaqx.dev.Dockerfile + (multi-hour CUDA-Q build the first time per pin) + --cuda-version V base image CUDA flavor (default: $CUDA_VERSION) + +Hardware: + --cuda-arch N CUDA architecture (default: auto via nvidia-smi; + DGX Spark GB10=121, GB200=100) + --roce-pair rxe SoftRoCE self-loop for the two-process cpu_roce + lanes (the default; no free ConnectX port pair + needed), OR + --roce-pair DEV0,DEV1 a real loopback-cabled IB device pair (DGX Spark), + OR --roce-pair none => those lanes SKIP + --fpga-device DEV ConnectX IB device facing the FPGA + (default: $FPGA_DEVICE, the GB200 lab wiring) + --bridge-ip IP server-side NIC IP (default $BRIDGE_IP) + --fpga-ip IP FPGA IP (default $FPGA_IP) + --page-size N RDMA ring slot size (default: 384, or 512 on + 64 KiB-page hosts; the device_graph lane rounds up + to the host-page-compatible value) + --no-fpga skip all FPGA lanes (e.g. Spark cabled in loopback + mode: the single cable is FPGA XOR loopback) + +Misc: + --hf-token-file FILE Hugging Face token for the gated Ising model (or + set HF_TOKEN); without a token the trt lanes fall + back to a staged bundle (see --artifacts-dir) or SKIP + --hf-token-prompt read the token from the terminal instead (nothing + written to disk; for shared/public accounts) + --keep-container leave the container running afterwards (debugging) + --lock-wait one hw CI run per machine at a time (the FPGA and + SoftRoCE objects are host-global): queue behind + the active run instead of failing fast + --force-unlock kill a wedged run that is still holding the host + lock, then proceed (a crashed run releases the + lock by itself; see the README) + --help, -h this help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --sha) SHA="$2"; shift ;; + --workdir) WORKDIR="$2"; shift ;; + --artifacts-dir) ARTIFACTS_DIR="$2"; shift ;; + --tier) TIER="$2"; shift ;; + --include-opt-in) INCLUDE_OPT_IN=true ;; + --only) ONLY_GLOB="$2"; shift ;; + --skip) SKIP_GLOB="$2"; shift ;; + --list) LIST_ONLY=true ;; + --strict) STRICT=true ;; + --base-image) BASE_IMAGE="$2"; shift ;; + --refresh-base) REFRESH_BASE=true ;; + --build-base) BUILD_BASE=true ;; + --cuda-version) CUDA_VERSION="$2"; shift ;; + --cuda-arch) CUDA_ARCH="$2"; shift ;; + --roce-pair) ROCE_PAIR="$2"; [[ "$ROCE_PAIR" == none ]] && ROCE_PAIR=""; shift ;; + --fpga-device) FPGA_DEVICE="$2"; shift ;; + --bridge-ip) BRIDGE_IP="$2"; shift ;; + --fpga-ip) FPGA_IP="$2"; shift ;; + --page-size) PAGE_SIZE="$2"; shift ;; + --no-fpga) NO_FPGA=true ;; + --hf-token-file) HF_TOKEN_FILE="$2"; shift ;; + --hf-token-prompt) HF_TOKEN_PROMPT=true ;; + --keep-container) KEEP_CONTAINER=true ;; + --lock-wait) LOCK_WAIT=true ;; + --force-unlock) FORCE_UNLOCK=true ;; + --help|-h) print_usage; exit 0 ;; + *) echo "ERROR: unknown option: $1" >&2; print_usage >&2; exit 1 ;; + esac + shift +done + +_info() { echo "[hw-ci] $*"; } +_err() { echo "[hw-ci] ERROR: $*" >&2; } +_die() { _err "$*"; exit 1; } + +[[ -n "$SHA" || "$LIST_ONLY" == true ]] || { print_usage >&2; _die "--sha is required"; } +case "$TIER" in examples|extra|all) ;; *) _die "--tier must be examples|extra|all" ;; esac + +# Interactive token entry: the token lives only in this process's memory (and +# the ising-prepare lane's docker exec env), never on disk -- for runs from +# shared/public accounts. +if [[ "$HF_TOKEN_PROMPT" == true && "$LIST_ONLY" != true ]]; then + [[ -r /dev/tty ]] || _die "--hf-token-prompt needs a terminal (use --hf-token-file or HF_TOKEN otherwise)" + read -rs -p "Hugging Face token (input hidden): " HF_TOKEN < /dev/tty; echo + [[ -n "$HF_TOKEN" ]] || _die "--hf-token-prompt: empty token" + export HF_TOKEN +fi + +# --------------------------------------------------------------------------- +# Host-wide run lock: one hw CI run per machine at a time. The FPGA (SIF +# registers/BRAM/ILA), the SoftRoCE objects (hwci-dummy0/hwci_rxe0), and the +# RoCE pair addressing are all host-global, so concurrent runs would corrupt +# each other. flock(2) is released by the kernel the instant the holding +# process dies (crash, Ctrl-C, kill -9), so no failure can leave a stale +# lock; the lock FILE persisting on disk is inert. The only lockout is a +# live-but-wedged holder -- --force-unlock kills whatever actually holds the +# lock (fuser sees the real holders, including orphaned children that +# inherited the fd), and the lock-busy message shows them. +# --------------------------------------------------------------------------- +LOCK_FILE="${HWCI_LOCK_FILE:-/tmp/cudaqx-hw-ci.lock}" +LOCK_FD="" + +acquire_run_lock() { + # 0666 regardless of umask: every user must be able to lock it. + ( umask 000; : >>"$LOCK_FILE" ) 2>/dev/null + exec {LOCK_FD}<>"$LOCK_FILE" || _die "cannot open lock file $LOCK_FILE" + + if ! flock -n "$LOCK_FD"; then + if [[ "$FORCE_UNLOCK" == true ]]; then + _info "--force-unlock: killing the current holder(s) of $LOCK_FILE:" + fuser -v "$LOCK_FILE" || true + fuser -k -TERM "$LOCK_FILE" >/dev/null 2>&1 || true + local i + for i in 1 2 3 4 5; do sleep 1; flock -n "$LOCK_FD" && break; done + if ! flock -n "$LOCK_FD"; then + fuser -k -KILL "$LOCK_FILE" >/dev/null 2>&1 || true + sleep 1 + fi + flock -n "$LOCK_FD" || _die "--force-unlock failed -- the holder likely belongs to another user; escalate with: sudo fuser -vk $LOCK_FILE" + elif [[ "$LOCK_WAIT" == true ]]; then + _info "another hw CI run is active ($(tr -d '\n' <"$LOCK_FILE" 2>/dev/null)); waiting for it to finish ..." + flock "$LOCK_FD" || _die "waiting for the lock failed" + else + local holder pid orphan="" + holder=$(tr -d '\n' <"$LOCK_FILE" 2>/dev/null) + pid=$(sed -n 's/.*pid=\([0-9]\+\).*/\1/p' <<<"$holder") + [[ -n "$pid" && ! -d "/proc/$pid" ]] && \ + orphan=" -- its recorded pid is gone, so an orphaned child still holds the lock (fuser -v $LOCK_FILE shows it)" + _err "another hw CI run is active on this machine: ${holder:-}$orphan" + _die "rerun with --lock-wait to queue behind it, or --force-unlock to kill a wedged holder" + fi + fi + # Advisory holder info for the messages above; the flock itself is the gate. + printf 'user=%s pid=%s sha=%s since=%s\n' "$USER" $$ "$SHA" "$(date -Is)" >"$LOCK_FILE" 2>/dev/null || true +} + +[[ "$LIST_ONLY" == true ]] || acquire_run_lock + +# --------------------------------------------------------------------------- +# Lane bookkeeping. Every lane lands in the summary exactly once as +# PASS / FAIL / SKIP(reason). Lanes run strictly sequentially: everything +# here shares one GPU, one FPGA, and one RoCE fabric. +# --------------------------------------------------------------------------- +LANE_NAMES=() +LANE_STATUS=() +LANE_DETAIL=() + +lane_selected() { + local name="$1" + case "$TIER" in + examples) [[ "$name" == examples/* || "$name" == optin/* ]] || return 1 ;; + extra) [[ "$name" == extra/* || "$name" == optin/* ]] || return 1 ;; + esac + [[ "$name" == optin/* && "$INCLUDE_OPT_IN" != true ]] && return 1 + # shellcheck disable=SC2053 + [[ -n "$ONLY_GLOB" && "$name" != $ONLY_GLOB ]] && return 1 + # shellcheck disable=SC2053 + [[ -n "$SKIP_GLOB" && "$name" == $SKIP_GLOB ]] && return 1 + return 0 +} + +record_lane() { # name status detail + LANE_NAMES+=("$1"); LANE_STATUS+=("$2"); LANE_DETAIL+=("$3") +} + +skip_lane() { # name reason + lane_selected "$1" || return 0 + if [[ "$LIST_ONLY" == true ]]; then echo " $1 [would SKIP: $2]"; return 0; fi + _info "SKIP $1: $2" + record_lane "$1" SKIP "$2" +} + +# run_lane NAME CMD -- CMD is a bash command line executed in the container. +# Extra `docker exec` flags (e.g. -e VAR=...) come from LANE_ENV. +LANE_ENV=() +run_lane() { + # Consume LANE_ENV first thing so a deselected/listed lane can never leak + # its env flags (or the HF token) into the next lane that runs. + local env_flags=(${LANE_ENV[@]+"${LANE_ENV[@]}"}) + LANE_ENV=() + local name="$1"; shift + lane_selected "$name" || return 0 + if [[ "$LIST_ONLY" == true ]]; then echo " $name"; return 0; fi + local log="$LOG_DIR/${name//\//_}.log" + _info "LANE $name" + local t0=$SECONDS rc=0 + docker exec ${env_flags[@]+"${env_flags[@]}"} "$CONTAINER" bash -lc "$*" \ + >"$log" 2>&1 || rc=$? + local dt=$((SECONDS - t0)) + if [[ $rc -eq 0 ]]; then + record_lane "$name" PASS "${dt}s" + _info "PASS $name (${dt}s)" + elif [[ $rc -eq 77 ]]; then + local reason + reason=$(grep -Eo 'SKIP[:(].*' "$log" | tail -1) + record_lane "$name" SKIP "${reason:-exit 77 (see $log)}" + _info "SKIP $name: ${reason:-exit 77}" + else + record_lane "$name" FAIL "rc=$rc log: $log" + _err "FAIL $name (rc=$rc) log: $log" + fi +} + +print_summary() { + local pass=0 fail=0 skip=0 i + echo + echo "================ HW-CI SUMMARY (sha ${SHORT_SHA:-?}) ================" + for i in "${!LANE_NAMES[@]}"; do + printf '%-5s %-48s %s\n' "${LANE_STATUS[$i]}" "${LANE_NAMES[$i]}" "${LANE_DETAIL[$i]}" + case "${LANE_STATUS[$i]}" in + PASS) ((pass++)) ;; FAIL) ((fail++)) ;; SKIP) ((skip++)) ;; + esac + done + echo "---------------------------------------------------------------" + echo "$pass passed, $fail failed, $skip skipped" + [[ $fail -gt 0 ]] && return 1 + if [[ "$STRICT" == true && $skip -gt 0 ]]; then + echo "--strict: treating $skip skip(s) as failure" + return 1 + fi + return 0 +} + +# --------------------------------------------------------------------------- +# Host-side setup +# --------------------------------------------------------------------------- +detect_cuda_arch() { + [[ -n "$CUDA_ARCH" ]] && return 0 + local cap + cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' .') + [[ -n "$cap" ]] || _die "cannot auto-detect the GPU (nvidia-smi); pass --cuda-arch" + CUDA_ARCH="$cap" + _info "CUDA architecture: sm_$CUDA_ARCH (auto-detected)" +} + +checkout_sha() { + SRC="$WORKDIR/src" + mkdir -p "$WORKDIR" + if [[ ! -d "$SRC/.git" ]]; then + _info "Cloning $REPO_URL" + git clone "$REPO_URL" "$SRC" || _die "clone failed" + fi + # --repo is authoritative on every run, not just the first: retarget the + # cached clone (a stale origin otherwise silently pins every later run + # to the first-ever --repo). + git -C "$SRC" remote set-url origin "$REPO_URL" + if git -C "$SRC" fetch origin "$SHA" 2>/dev/null; then + # FETCH_HEAD is exactly the requested branch/tag/SHA tip; never + # resolve "$SHA" locally here or a branch name would silently hit a + # stale local ref from clone time. + git -C "$SRC" checkout --detach FETCH_HEAD || _die "cannot check out '$SHA'" + else + # Not fetchable by name (e.g. an abbreviated SHA): fetch everything, + # then the ref must resolve -- otherwise fail loudly instead of + # falling back to an arbitrary FETCH_HEAD. + git -C "$SRC" fetch origin || _die "fetch failed" + git -C "$SRC" checkout --detach "$SHA" 2>/dev/null || _die "cannot check out '$SHA'" + fi + SHORT_SHA=$(git -C "$SRC" rev-parse --short=12 HEAD) + _info "Testing cudaqx @ $SHORT_SHA" +} + +platform() { + case "$(uname -m)" in + aarch64|arm64) echo arm64 ;; + x86_64) echo amd64 ;; + *) _die "unsupported platform: $(uname -m)" ;; + esac +} + +resolve_base_image() { + [[ -n "$BASE_IMAGE" ]] && { _info "Base image (override): $BASE_IMAGE"; return 0; } + local plat shortref candidate fallback + plat=$(platform) + shortref=$(jq -r '.cudaq.ref' "$SRC/.cudaq_version" | head -c8) + candidate="ghcr.io/nvidia/cudaqx-dev:${shortref}-${plat}-cu${CUDA_VERSION}" + fallback="ghcr.io/nvidia/cudaqx-dev:latest-${plat}-cu${CUDA_VERSION}" + if docker image inspect "$candidate" >/dev/null 2>&1 || docker pull "$candidate" >/dev/null 2>&1; then + BASE_IMAGE="$candidate" + elif [[ "$BUILD_BASE" == true ]]; then + _info "Pin-matched base $candidate unavailable; building it locally (--build-base)" + docker build -f "$SRC/docker/build_env/cudaqx.dev.Dockerfile" \ + --build-arg base_image="ghcr.io/nvidia/cuda-quantum-devcontainer:${plat}-cu${CUDA_VERSION%%.*}.${CUDA_VERSION#*.}-gcc12-main" \ + --build-arg cuda_version="$CUDA_VERSION" \ + -t "$candidate" "$SRC" || _die "local base build failed" + BASE_IMAGE="$candidate" + elif docker image inspect "$fallback" >/dev/null 2>&1 || docker pull "$fallback" >/dev/null 2>&1; then + _info "Pin-matched base $candidate unavailable; falling back to $fallback" + BASE_IMAGE="$fallback" + else + _die "no usable base image: tried $candidate and $fallback (see --build-base / --base-image)" + fi + _info "Base image: $BASE_IMAGE" +} + +build_image() { + IMAGE="cudaqx-decoding-hwci:$SHORT_SHA" + local pull_flag=() + [[ "$REFRESH_BASE" == true ]] && pull_flag=(--pull) + _info "Building dev image $IMAGE (cached layers reused when unchanged)" + docker build ${pull_flag[@]+"${pull_flag[@]}"} \ + -f "$SRC/docker/decoding-server/dev.Dockerfile" \ + --build-arg base_image="$BASE_IMAGE" \ + --build-arg cuda_native_arch="$CUDA_ARCH" \ + -t "$IMAGE" "$SRC/docker/decoding-server" || _die "docker build failed" +} + +check_cudaq_pin() { + local image_ref src_ref + image_ref=$(docker run --rm "$IMAGE" jq -r '.cudaq.ref' /cudaq_version 2>/dev/null) + src_ref=$(jq -r '.cudaq.ref' "$SRC/.cudaq_version") + if [[ -z "$image_ref" || "$image_ref" == "null" ]]; then + _info "WARNING: image has no /cudaq_version; skipping the stale-pin check" + return 0 + fi + if [[ "$image_ref" != "$src_ref" ]]; then + _die "stale base image: it bakes CUDA-Q $image_ref but the commit pins $src_ref. +Wait for build_dev.yaml to publish the new pin's image, pass --base-image, or use --build-base." + fi + _info "CUDA-Q pin check OK ($src_ref)" +} + +start_container() { + # $USER in the name so a --keep-container debugging container from one + # user is not docker-rm'd by another user later testing the same SHA + # (concurrent runs are already excluded by the host-wide lock). + CONTAINER="hwci-$USER-$SHORT_SHA" + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + mkdir -p "$WORKDIR/ccache" + local artifacts_mount=() + if [[ -d "$ARTIFACTS_DIR" ]]; then + artifacts_mount=(-v "$ARTIFACTS_DIR:/artifacts:ro") + else + _info "Artifacts dir $ARTIFACTS_DIR absent; proprietary lanes will SKIP" + fi + # rxe mode may need to build the OFED-compat rdma_rxe module in-container + # (see setup_roce_pair), which compiles against the HOST kernel headers + # and ofa_kernel tree. + local rxe_mount=() + if [[ "$ROCE_PAIR" == "rxe" ]]; then + rxe_mount=(-v /lib/modules:/lib/modules:ro -v /usr/src:/usr/src:ro) + fi + # /dev/infiniband is a live bind mount, NOT --device: --device snapshots + # the char devices at container creation, so the uverbs node of an rxe + # device created later by setup_roce_pair would never appear in the + # container and libibverbs would not find the device. The container is + # privileged, so no device-cgroup allowance is lost by the switch. + docker run -d --name "$CONTAINER" \ + --privileged --net=host --gpus all --shm-size=8g \ + --ulimit memlock=-1:-1 \ + -v /dev/infiniband:/dev/infiniband \ + -v "$SRC:/workspaces/cudaqx" \ + -v "$WORKDIR/ccache:/root/.ccache" \ + ${artifacts_mount[@]+"${artifacts_mount[@]}"} \ + ${rxe_mount[@]+"${rxe_mount[@]}"} \ + "$IMAGE" sleep infinity >/dev/null || _die "docker run failed" +} + +cleanup() { + [[ "$KEEP_CONTAINER" == true ]] && return 0 + if [[ -n "${CONTAINER:-}" ]]; then + teardown_roce_pair 2>/dev/null + docker rm -f "$CONTAINER" >/dev/null 2>&1 + fi +} +trap cleanup EXIT + +in_ctr() { docker exec "$CONTAINER" bash -lc "$*"; } + +# --------------------------------------------------------------------------- +# cpu_roce endpoint pair. +# rxe: SoftRoCE self-loop on a dummy netdev -- both endpoints share +# one rxe device/IP (the pattern documented in the in-tree +# surface_code-1-cqr-two-process-test.sh). Needs rdma_rxe on +# the HOST kernel. +# DEV0,DEV1: real loopback-cabled ConnectX pair (e.g. DGX Spark): +# 10.0.0.1/24 <-> 10.0.0.2/24 with permanent neighbor entries +# (same-host IPs otherwise resolve via lo and RDMA CM times out). +# All netlink/rdma calls run inside the privileged --net=host container, so +# they act on the host netns; objects carry the hwci- prefix for teardown. +# --------------------------------------------------------------------------- +ROCE_READY=false +CH_DEV=""; CH_IP=""; DA_DEV=""; DA_IP="" +NET0=""; NET1="" + +apply_roce_pair_addrs() { + # Idempotent; called before every cpu_roce lane as well as at setup: + # NetworkManager-managed ports drop statically added addresses on each + # DHCP retry cycle (see the README for the permanent nmcli fix). + in_ctr " + set -e + ip link set $NET0 up; ip link set $NET1 up + ip addr replace 10.0.0.1/24 dev $NET0 + ip addr replace 10.0.0.2/24 dev $NET1 + mac0=\$(cat /sys/class/net/$NET0/address) + mac1=\$(cat /sys/class/net/$NET1/address) + ip neigh replace 10.0.0.2 lladdr \$mac1 nud permanent dev $NET0 + ip neigh replace 10.0.0.1 lladdr \$mac0 nud permanent dev $NET1 + " +} + +wait_roce_gids() { + # The IPv4-mapped RoCE GIDs (::ffff:10.0.0.x) appear asynchronously + # after the address add, and the transceivers refuse to start without + # them. Host sysfs and the --net=host container see the same tables. + local i + for i in $(seq 1 20); do + grep -qs 'ffff:0a00:0001' "/sys/class/infiniband/$CH_DEV/ports/1/gids/"* && \ + grep -qs 'ffff:0a00:0002' "/sys/class/infiniband/$DA_DEV/ports/1/gids/"* && return 0 + sleep 0.5 + done + return 1 +} + +setup_roce_pair() { + [[ -z "$ROCE_PAIR" ]] && return 0 + if [[ "$ROCE_PAIR" == "rxe" ]]; then + if ! grep -qw rdma_rxe /proc/modules; then + sudo -n modprobe rdma_rxe 2>/dev/null || true + fi + if ! grep -qw rdma_rxe /proc/modules; then + # DOCA/MLNX-OFED DKMS hosts: the distro rdma_rxe cannot bind to + # the OFED ib_core (symbol CRC mismatch), so build the staged + # OFED-compat copy against the host headers mounted at + # /lib/modules + /usr/src and load it from the privileged + # container (see rxe-ofed/README.md). + _info "distro rdma_rxe not loadable; building the OFED-compat module in-container" + in_ctr "make -C /opt/rxe-ofed/src >/tmp/rxe-ofed-build.log 2>&1 \ + && insmod /opt/rxe-ofed/src/rdma_rxe.ko" || { + _err "OFED-compat rdma_rxe build/load failed; last lines of the build log:" + in_ctr "tail -15 /tmp/rxe-ofed-build.log" >&2 || true + _err "(full log: docker exec $CONTAINER cat /tmp/rxe-ofed-build.log)" + return 1 + } + fi + in_ctr " + ip link add hwci-dummy0 type dummy 2>/dev/null || true + ip addr replace 10.88.0.1/24 dev hwci-dummy0 + ip link set hwci-dummy0 up + rdma link show hwci_rxe0 >/dev/null 2>&1 || \ + rdma link add hwci_rxe0 type rxe netdev hwci-dummy0 + ibv_devinfo -d hwci_rxe0 >/dev/null + " || { _err "SoftRoCE setup failed. Note: if 'rdma link' shows the device but ibv_devinfo cannot open it, the image's Mellanox-OFED ibverbs-providers lacks the rxe userspace provider (a known gap; see the README)"; return 1; } + CH_DEV=hwci_rxe0; CH_IP=10.88.0.1 + DA_DEV=hwci_rxe0; DA_IP=10.88.0.1 + else + local dev0="${ROCE_PAIR%%,*}" dev1="${ROCE_PAIR##*,}" + [[ -n "$dev0" && -n "$dev1" && "$dev0" != "$dev1" ]] \ + || { _err "--roce-pair expects rxe or DEV0,DEV1"; return 1; } + # Resolve ibdev -> netdev on the host via sysfs: the image has no + # ibdev2netdev, and --net=host keeps the names identical inside the + # container anyway. + NET0=$(ls "/sys/class/infiniband/$dev0/device/net" 2>/dev/null | head -1) + NET1=$(ls "/sys/class/infiniband/$dev1/device/net" 2>/dev/null | head -1) + [[ -n "$NET0" && -n "$NET1" ]] \ + || { _err "cannot resolve netdevs for $ROCE_PAIR (see /sys/class/infiniband)"; return 1; } + CH_DEV="$dev0"; CH_IP=10.0.0.1 + DA_DEV="$dev1"; DA_IP=10.0.0.2 + apply_roce_pair_addrs \ + || { _err "RoCE pair setup failed for $ROCE_PAIR"; return 1; } + wait_roce_gids \ + || { _err "IPv4 RoCE GIDs did not appear on $dev0/$dev1"; return 1; } + fi + ROCE_READY=true + _info "cpu_roce pair ready: channel=$CH_DEV/$CH_IP daemon=$DA_DEV/$DA_IP" +} + +teardown_roce_pair() { + [[ "$ROCE_PAIR" == "rxe" && -n "${CONTAINER:-}" ]] || return 0 + in_ctr " + rdma link delete hwci_rxe0 2>/dev/null || true + ip link delete hwci-dummy0 2>/dev/null || true + " || true +} + +roce_env() { # docker exec env flags for the cpu_roce topology + # Re-assert the pair right before each lane: on NetworkManager-managed + # ports the addresses vanish on NM's retry timer, which killed lanes + # minutes after a successful setup. + if [[ "$LIST_ONLY" != true && "$ROCE_PAIR" != rxe && "$ROCE_READY" == true ]]; then + apply_roce_pair_addrs >/dev/null 2>&1 && wait_roce_gids \ + || _info "WARNING: cpu_roce pair re-assert failed; lane may fail" + fi + LANE_ENV+=( -e "CUDAQ_CPU_ROCE_TEST_CHANNEL_DEVICE=$CH_DEV" + -e "CUDAQ_CPU_ROCE_TEST_CHANNEL_IP=$CH_IP" + -e "CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE=$DA_DEV" + -e "CUDAQ_CPU_ROCE_TEST_DAEMON_IP=$DA_IP" ) +} + +# --------------------------------------------------------------------------- +# Page-size geometry. cpu_roce host dispatch has no host-page constraint; +# the device_graph ring (64 slots) must total a multiple of the host page +# size, so its value rounds up to the compatible one. +# --------------------------------------------------------------------------- +# Lane label: the trt_decoder plugin runs the exported Ising model in these +# lanes -- make that visible in lane names and the summary table. The +# --decoder argument stays 'trt_decoder' (the plugin name). +lane_label() { [[ "$1" == trt_decoder ]] && echo 'trt_decoder(ising)' || echo "$1"; } + +derive_page_sizes() { + local host_page; host_page=$(getconf PAGESIZE) + if [[ -z "$PAGE_SIZE" ]]; then + PAGE_SIZE=384 + [[ "$host_page" -gt 4096 ]] && PAGE_SIZE=512 + fi + local stride=$(( host_page / 64 )) + PAGE_SIZE_DG=$(( (PAGE_SIZE + stride - 1) / stride * stride )) + _info "page size: $PAGE_SIZE (device_graph ring: $PAGE_SIZE_DG; host page $host_page)" +} + +# --------------------------------------------------------------------------- +# Lane definitions +# --------------------------------------------------------------------------- +DEMO=/workspaces/cudaqx/docs/sphinx/examples/qec/realtime_decoding_demo +DEMO_ARGS="--install-prefix /usr/local/cudaqx --cudaq-prefix /usr/local/cudaq \ +--realtime-lib-dir /tmp/cudaq-realtime --example-build-dir /workspaces/cudaqx/demo-build" +NV_QLDPC_PLUGIN_HOST="$ARTIFACTS_DIR/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so" +CUDEVICE_HOST="$ARTIFACTS_DIR/cudevice/libcudaq-qec-realtime-cudevice-proprietary.a" +NV_QLDPC_PLUGIN_CTR=/artifacts/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so +CUDEVICE_CTR=/artifacts/cudevice/libcudaq-qec-realtime-cudevice-proprietary.a +ISING_BUNDLE=/tmp/ising-bundle +# Optional pre-built bundle for tokenless machines (see --hf-token-file help): +# copied into the container so the trt lanes run; only the HF download/export +# path loses coverage, and the ising-prepare SKIP reason says so. +ISING_STAGED_HOST="$ARTIFACTS_DIR/ising-bundle" +ISING_STAGED_CTR=/artifacts/ising-bundle +CQ_SRC=/tmp/cudaq-realtime-src + +have_hf_token() { + [[ "$LIST_ONLY" == true ]] && return 0 # --list is host-independent + [[ -n "${HF_TOKEN:-}" ]] && return 0 + [[ -n "$HF_TOKEN_FILE" && -r "$HF_TOKEN_FILE" ]] && return 0 + return 1 +} + +hf_token() { + if [[ -n "${HF_TOKEN:-}" ]]; then echo "$HF_TOKEN"; else cat "$HF_TOKEN_FILE"; fi +} + +ising_ready() { + [[ "$LIST_ONLY" == true ]] && return 0 + in_ctr "test -f $ISING_BUNDLE/metadata.txt" 2>/dev/null +} + +# ctest lanes: `ctest -R` exits 0 when NOTHING matches, which would +# false-PASS a lane whose test was never registered (configure-time gates). +# Count first and convert "no match" into a named SKIP. +ctest_cmd() { # regex reason-when-unregistered + local regex="$1" reason="$2" + # --timeout only applies to tests without their own TIMEOUT property; it + # bounds a wedged test at 15 min instead of ctest's 1500 s default. + echo "cd /workspaces/cudaqx/build && \ +n=\$(ctest -N -R '$regex' 2>/dev/null | sed -n 's/^Total Tests: //p'); \ +if [ \"\${n:-0}\" -eq 0 ]; then echo 'SKIP: $reason'; exit 77; fi; \ +ctest --output-on-failure --timeout 900 -R '$regex'" +} + +fpga_dev_flag() { [[ -n "$FPGA_DEVICE" ]] && echo "--device $FPGA_DEVICE"; } + +run_examples_tier() { + local d + + # -- Ising bundle: fresh gated-HF download + export when a token exists; + # otherwise fall back to a bundle staged in the artifacts dir. The copy + # runs outside the lane gate so `--only .../trt_decoder` also benefits. + if ! have_hf_token && [[ "$LIST_ONLY" != true && -f "$ISING_STAGED_HOST/metadata.txt" ]]; then + in_ctr "rm -rf $ISING_BUNDLE && cp -r $ISING_STAGED_CTR $ISING_BUNDLE" \ + || _info "WARNING: staged Ising bundle copy failed" + fi + if have_hf_token; then + [[ "$LIST_ONLY" == true ]] || LANE_ENV=( -e "HF_TOKEN=$(hf_token)" ) + run_lane "examples/ising-prepare" " + set -e + rm -rf $ISING_BUNDLE + app=\$(find /workspaces/cudaqx/build -name surface_code-4-yaml -type f -perm -u+x | head -1) + [ -n \"\$app\" ] || { echo 'surface_code-4-yaml generator not in the build tree'; exit 1; } + python3 $DEMO/prepare_ising_artifacts.py prepare \ + --app \"\$app\" --artifacts-dir $ISING_BUNDLE --yes" + elif ising_ready; then + skip_lane "examples/ising-prepare" "no HF token; trt lanes use the staged bundle from $ISING_STAGED_HOST (HF download/export path NOT exercised)" + else + skip_lane "examples/ising-prepare" "no HF token (--hf-token-file/--hf-token-prompt/HF_TOKEN) and no staged bundle at $ISING_STAGED_HOST" + fi + + # -- qpu-kernel over udp: the no-hardware baseline ----------------------- + for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do + local name="examples/qpu-kernel/udp/$(lane_label "$d")" extra="" + case "$d" in + nv-qldpc-decoder) + [[ -f "$NV_QLDPC_PLUGIN_HOST" || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST"; continue; } + extra="--nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0" ;; + trt_decoder) + ising_ready || { skip_lane "$name" "no Ising bundle (ising-prepare failed/skipped)"; continue; } + extra="--ising-artifacts-dir $ISING_BUNDLE" ;; + esac + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source qpu-kernel --decoder $d $DEMO_ARGS $extra" + done + + # -- qpu-kernel over cpu_roce: real RDMA verbs on the RoCE pair ---------- + # No --setup-network: the runner configured the pair itself (the demo's + # helper resolves ports via ibdev2netdev, which cannot see rxe devices). + for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do + local name="examples/qpu-kernel/cpu_roce/$(lane_label "$d")" extra="" + [[ "$ROCE_READY" == true || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "no cpu_roce pair (--roce-pair not set / setup failed)"; continue; } + case "$d" in + nv-qldpc-decoder) + [[ -f "$NV_QLDPC_PLUGIN_HOST" || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST"; continue; } + extra="--nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0" ;; + trt_decoder) + ising_ready || { skip_lane "$name" "no Ising bundle (ising-prepare failed/skipped)"; continue; } + extra="--ising-artifacts-dir $ISING_BUNDLE" ;; + esac + roce_env + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source qpu-kernel --wire cpu_roce --decoder $d $DEMO_ARGS $extra" + done + + # -- FPGA source, cpu_roce wire, host dispatch --------------------------- + for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do + local name="examples/fpga/cpu_roce/$(lane_label "$d")" extra="" + [[ "$NO_FPGA" == true ]] && { skip_lane "$name" "--no-fpga"; continue; } + case "$d" in + nv-qldpc-decoder) + [[ -f "$NV_QLDPC_PLUGIN_HOST" || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST"; continue; } + extra="--nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0" ;; + trt_decoder) + ising_ready || { skip_lane "$name" "no Ising bundle (ising-prepare failed/skipped)"; continue; } + extra="--ising-artifacts-dir $ISING_BUNDLE" ;; + esac + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source fpga --dispatch host --decoder $d \ + --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE $DEMO_ARGS $extra" + done + + # -- FPGA source, device_graph dispatch (nv-qldpc only) ------------------ + local name="examples/fpga/device-graph/nv-qldpc-decoder" + if [[ "$NO_FPGA" == true ]]; then + skip_lane "$name" "--no-fpga" + elif [[ ! -f "$NV_QLDPC_PLUGIN_HOST" && "$LIST_ONLY" != true ]]; then + skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST" + elif [[ ! -f "$CUDEVICE_HOST" && "$LIST_ONLY" != true ]]; then + skip_lane "$name" "missing $CUDEVICE_HOST (device_graph dispatch not built)" + else + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source fpga --decoder nv-qldpc-decoder \ + --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE_DG --nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0 $DEMO_ARGS" + fi +} + +run_extra_tier() { + local hsb=/workspaces/cudaqx/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh + + # -- two-process device_call channel over real RDMA verbs ---------------- + if [[ "$ROCE_READY" == true || "$LIST_ONLY" == true ]]; then + LANE_ENV=( -e "QEC_DECODING_SERVER_TRANSPORT=cpu_roce" ); roce_env + run_lane "extra/ctest/two-process-cpu-roce" \ + "$(ctest_cmd 'DecodingServerTwoProcess' 'DecodingServerTwoProcess tests not registered')" + LANE_ENV=( -e "QEC_DECODING_SERVER_TRANSPORT=cpu_roce" ); roce_env + run_lane "extra/ctest/app-two-process-cpu-roce" \ + "$(ctest_cmd 'app_examples.surface_code-1-cqr-two-process' 'two-process app tests not registered')" + else + skip_lane "extra/ctest/two-process-cpu-roce" "no cpu_roce pair (--roce-pair not set / setup failed)" + skip_lane "extra/ctest/app-two-process-cpu-roce" "no cpu_roce pair (--roce-pair not set / setup failed)" + fi + + # -- decoding_server over the FPGA (HSB control plane + SIF playback) ---- + if [[ "$NO_FPGA" == true ]]; then + skip_lane "extra/hsb-fpga-server/cpu_roce" "--no-fpga" + skip_lane "extra/hsb-fpga-server/gpu_roce" "--no-fpga" + else + run_lane "extra/hsb-fpga-server/cpu_roce" \ + "bash $hsb --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE --cuda-quantum-dir $CQ_SRC" + if [[ ( -f "$NV_QLDPC_PLUGIN_HOST" && -f "$CUDEVICE_HOST" ) || "$LIST_ONLY" == true ]]; then + run_lane "extra/hsb-fpga-server/gpu_roce" \ + "bash $hsb --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE_DG --transport gpu_roce --decoder nv-qldpc-decoder \ + --nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --cuda-quantum-dir $CQ_SRC" + elif [[ ! -f "$NV_QLDPC_PLUGIN_HOST" ]]; then + skip_lane "extra/hsb-fpga-server/gpu_roce" "missing $NV_QLDPC_PLUGIN_HOST" + else + skip_lane "extra/hsb-fpga-server/gpu_roce" "missing $CUDEVICE_HOST (device_graph dispatch not built)" + fi + fi + + # -- GB200-class GPU ctests unreachable in normal CI ---------------------- + run_lane "extra/ctest/qldpc-graph" \ + "$(ctest_cmd '^test_realtime_qldpc_graph_decoding$' 'not registered (needs nv-qldpc plugin + cudevice archive at configure)')" + run_lane "extra/ctest/mixed-dispatch" \ + "$(ctest_cmd 'app_examples.surface_code-4-yaml-mixed-dispatch' 'not registered (needs cudevice archive at configure)')" + + # -- gpu_roce bridge cross-check (same data plane, no server layers) ------ + local gbridge=/workspaces/cudaqx/libs/qec/unittests/utils/gpu_roce_qldpc_graph_decoder_test.sh + if [[ "$NO_FPGA" == true ]]; then + skip_lane "extra/gpu-roce-qldpc-bridge" "--no-fpga" + elif [[ ( ! -f "$NV_QLDPC_PLUGIN_HOST" || ! -f "$CUDEVICE_HOST" ) && "$LIST_ONLY" != true ]]; then + skip_lane "extra/gpu-roce-qldpc-bridge" "missing nv-qldpc plugin and/or cudevice archive" + else + # --spacing 100: at the playback tool's default 10 us inter-shot + # spacing the ILA verification deterministically undercounts + # (194/500 on the Spark); 100 us matches the demo lanes' pacing. + run_lane "extra/gpu-roce-qldpc-bridge" " + set -e + bridge=\$(find /workspaces/cudaqx/build -name gpu_roce_qldpc_graph_decoder_bridge -type f | head -1) + [ -n \"\$bridge\" ] || { echo 'SKIP: bridge executable not built'; exit 77; } + bash $gbridge --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE_DG --spacing 100 \ + --cuda-qx-dir /workspaces/cudaqx --cuda-quantum-dir $CQ_SRC \ + --hsb-dir /opt/holoscan-sensor-bridge \ + --proprietary-archive $CUDEVICE_CTR --nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR" + fi +} + +run_optin_tier() { + # The predecoder bridge links the experimental cudaq-realtime-pipeline + # library (CUDAQX_QEC_ENABLE_REALTIME_PIPELINE, OFF by default and + # pending a port to the post-PR4770 graph-launch API), so today this + # lane documents the coverage gap rather than exercising it. + if [[ "$NO_FPGA" == true ]]; then + skip_lane "optin/gpu-roce-predecoder" "--no-fpga" + return 0 + fi + run_lane "optin/gpu-roce-predecoder" " + bridge=\$(find /workspaces/cudaqx/build -name gpu_roce_predecoder_bridge -type f | head -1) + [ -n \"\$bridge\" ] || { echo 'SKIP: bridge not built (experimental realtime_pipeline)'; exit 77; } + bash /workspaces/cudaqx/libs/qec/unittests/realtime/gpu_roce_predecoder_test.sh \ + --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP" +} + +run_all_lanes() { + run_examples_tier + run_extra_tier + run_optin_tier +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +if [[ "$LIST_ONLY" == true ]]; then + ROCE_READY=true # listing shows the full lane set, not this host's skips + PAGE_SIZE=${PAGE_SIZE:-384}; PAGE_SIZE_DG=$PAGE_SIZE + echo "Lanes for --tier $TIER$([[ "$INCLUDE_OPT_IN" == true ]] && echo ' --include-opt-in'):" + run_all_lanes + exit 0 +fi + +detect_cuda_arch +derive_page_sizes +checkout_sha +LOG_DIR="$WORKDIR/logs/$SHORT_SHA" +mkdir -p "$LOG_DIR" +resolve_base_image +build_image +check_cudaq_pin +start_container +setup_roce_pair || _info "continuing without a cpu_roce pair" + +_info "Image ready; compiling the commit under test inside the container:" +_info " cudaq-realtime + cudaqx + demo binaries (log: $LOG_DIR/container_build.log)" +build_log="$LOG_DIR/container_build.log" +if ! in_ctr "bash /workspaces/cudaqx/docker/decoding-server/hw_ci/container_build.sh \ + --cuda-arch $CUDA_ARCH" >"$build_log" 2>&1; then + tail -40 "$build_log" >&2 + _die "in-container source build failed; full log: $build_log" +fi +_info "Source build complete (cudaq-realtime + cudaqx + demo)" + +run_all_lanes +print_summary diff --git a/docker/decoding-server/hw_ci/rxe-ofed/README.md b/docker/decoding-server/hw_ci/rxe-ofed/README.md new file mode 100644 index 000000000..782288cd2 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/README.md @@ -0,0 +1,64 @@ +# SoftRoCE (rxe) on MLNX/DOCA-OFED hosts + +Two independent gaps stop `--roce-pair rxe` from working out of the box on +hosts that run DOCA-OFED (validated on a GB200, `6.17.0-1008-nvidia-64k` + +DOCA-OFED 25.10): + +1. **Kernel:** the OFED DKMS stack replaces `ib_core` (in + `/lib/modules/*/updates/dkms/`), whose exported symbol CRCs differ from + the in-tree build, so the distro's `rdma_rxe.ko` fails to load with + `disagrees about version of symbol ib_*` (err -22) — and MLNX OFED + dropped the rxe driver from its own source tree (only the uapi header + remains in `ofa_kernel`). +2. **Userspace:** the Mellanox rdma-core fork's `ibverbs-providers` ships + only the mlx5 provider. Its provider ABI (`rdmav59`) differs from + Ubuntu's rdma-core, so the Ubuntu package can't supply `librxe` either; + in their source release the rxe provider exists but is `if (0)`-disabled + in `CMakeLists.txt`. + +## What this directory provides (gap 1) + +- `ofed-compat.patch` — three small deltas that make the **upstream v6.17** + `drivers/infiniband/sw/rxe` compile against the OFED compat API + (GPL-2.0, like the sources it patches): + - drop the `struct ib_dmah *` parameter from `rxe_reg_user_mr` + (upstream v6.16+ API; OFED's `ib_device_ops` predates it), + - `umem_odp->map.pfn_list` → `umem_odp->pfn_list` (OFED keeps the + pre-`hmm_dma_map` layout), + - (via `rxe_ofed_compat.h`, injected by the makefile with `-include`) + the two ODP capability bits OFED's enum lacks. +- `makefile` — external-module build against + `/usr/src/ofa_kernel[-dkms]//` (include chain + + `KBUILD_EXTRA_SYMBOLS`, the same pattern MLNX's own iser/isert DKMS + packages use). Refuses to build without an OFED tree: on inbox-rdma + hosts the distro `modprobe rdma_rxe` is the right module. +- `prepare-src.sh` — sparse-clones the pinned upstream tag (default + `v6.17`, image build-arg `rxe_kernel_ref`), applies the patch + (`--fuzz=0`: drift fails the image build), and drops the makefile + + compat header next to the sources. + +`dev.Dockerfile` stages the patched source at `/opt/rxe-ofed/src` at image +build. The `.ko` itself is **per-host** and is built at container setup +time by `run_hw_ci.sh` (rxe mode mounts `/usr/src` + `/lib/modules` +read-only, builds in-container, and `insmod`s from the privileged +container). Nothing is loaded on hosts where the distro module works — +that is always tried first. + +Gap 2 is fixed directly in `dev.Dockerfile`: it builds `librxe-rdmav` +from the **same Mellanox rdma-core source release** as the installed +packages (the `MLNX_OFED_SRC-debian` bundle from the DOCA `SOURCES` repo, +sha256-pinned) and installs it into the image's libibverbs provider +directory. + +## Caveats + +- The loaded module does not survive a host reboot; the runner re-builds + and re-loads on demand. For a permanent host install, DKMS-ify this + directory (out of scope here). +- The pinned kernel ref should roughly match the host kernel generation; + the OFED tree it compiles against, however, is whatever the host has + installed. A very different host kernel will surface as loud compile + errors at setup time (the lanes then SKIP, never silently). +- If a future DOCA release restores rxe (kernel or userspace), the distro + `modprobe` / packaged provider win automatically and this machinery goes + dormant. diff --git a/docker/decoding-server/hw_ci/rxe-ofed/makefile b/docker/decoding-server/hw_ci/rxe-ofed/makefile new file mode 100644 index 000000000..c7220ac28 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/makefile @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Build the (patched) upstream rxe SoftRoCE driver as an external module +# against the MLNX/DOCA-OFED ofa_kernel compat headers + Module.symvers, so +# it binds to the OFED DKMS ib_core instead of the in-tree one. +# +# Why: on hosts with doca-ofed / mlnx-ofed-kernel-dkms installed (e.g. +# GB200), ib_core & friends come from /lib/modules/*/updates/dkms and export +# different symbol CRCs than the in-tree build, so the distro's rdma_rxe.ko +# fails to load with "disagrees about version of symbol ib_*" -- and OFED +# dropped the rxe driver from its own source tree. The external-module +# pattern below (OFED include chain + KBUILD_EXTRA_SYMBOLS) is copied from +# the makefiles of MLNX OFED's own out-of-tree DKMS packages (iser/isert). +# +# This makefile, rxe_ofed_compat.h and ofed-compat.patch are expected to sit +# next to (and the patch already applied to) the upstream +# drivers/infiniband/sw/rxe sources -- prepare-src.sh stages exactly that. +# GNU make prefers this lowercase 'makefile'; kbuild's sub-make reads the +# upstream capital 'Makefile' for the object list. + +KVER ?= $(shell uname -r) +ARCH_M := $(shell uname -m) +OFA_DIR ?= $(shell test -d /usr/src/ofa_kernel-dkms/$(ARCH_M) && echo /usr/src/ofa_kernel-dkms/$(ARCH_M) || echo /usr/src/ofa_kernel/$(ARCH_M)) +OFA := $(OFA_DIR)/$(KVER) +K_BUILD ?= /lib/modules/$(KVER)/build + +ifneq ($(wildcard $(OFA)/Module.symvers),) +include $(OFA)/configure.mk.kernel +endif + +autoconf_h := $(shell /bin/ls -1 $(K_BUILD)/include/*/autoconf.h 2>/dev/null | head -1) +kconfig_h := $(shell /bin/ls -1 $(K_BUILD)/include/*/kconfig.h 2>/dev/null | head -1) +ifneq ($(kconfig_h),) +KCONFIG_H = -include $(kconfig_h) +endif + +LINUXINCLUDE=\ + -include $(autoconf_h) \ + $(KCONFIG_H) \ + -include $(OFA)/include/linux/compat-2.6.h \ + -include $(CURDIR)/rxe_ofed_compat.h \ + -I$(OFA)/include \ + -I$(OFA)/include/uapi \ + $(BACKPORT_INCLUDES) \ + -I$$(srctree)/arch/$$(SRCARCH)/include \ + -I$$(objtree)/arch/$$(SRCARCH)/include/generated \ + -I$$(objtree)/include \ + -I$$(srctree)/arch/$$(SRCARCH)/include/uapi \ + -I$$(objtree)/arch/$$(SRCARCH)/include/generated/uapi \ + -I$$(srctree)/include \ + -I$$(srctree)/include/uapi \ + -I$$(objtree)/include/generated/uapi \ + # + +default: +ifneq ($(wildcard $(OFA)/Module.symvers),) + $(MAKE) -C $(K_BUILD) O=$(K_BUILD) M=$(CURDIR) \ + KBUILD_EXTRA_SYMBOLS=$(OFA)/Module.symvers \ + CONFIG_RDMA_RXE=m \ + LINUXINCLUDE='$(LINUXINCLUDE)' \ + modules +else + @echo "error: $(OFA_DIR)/$(KVER) has no Module.symvers." >&2; \ + echo "This OFED-compat build is only for MLNX/DOCA-OFED DKMS hosts;" >&2; \ + echo "on inbox-rdma hosts load the distro module: sudo modprobe rdma_rxe" >&2; \ + echo "(if run from the hw_ci container: are /usr/src and /lib/modules" >&2; \ + echo "mounted from the host?)" >&2; \ + exit 1 +endif + +clean: + rm -f *.o *.ko *.mod *.mod.c modules.order Module.symvers .*.cmd diff --git a/docker/decoding-server/hw_ci/rxe-ofed/ofed-compat.patch b/docker/decoding-server/hw_ci/rxe-ofed/ofed-compat.patch new file mode 100644 index 000000000..fe68ed0e5 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/ofed-compat.patch @@ -0,0 +1,81 @@ +--- a/rxe_verbs.c 2026-08-11 20:26:52.811000000 +0000 ++++ b/rxe_verbs.c 2026-08-11 20:16:55.914000000 +0000 +@@ -1269,9 +1269,11 @@ + return ERR_PTR(err); + } + ++/* OFED-compat: DOCA-OFED 25.10's ib_device_ops.reg_user_mr has no ++ * struct ib_dmah parameter (added upstream in v6.16+); drop it and the ++ * corresponding reject-if-set check from the v6.17 source. */ + static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start, + u64 length, u64 iova, int access, +- struct ib_dmah *dmah, + struct ib_udata *udata) + { + struct rxe_dev *rxe = to_rdev(ibpd->device); +@@ -1279,9 +1281,6 @@ + struct rxe_mr *mr; + int err, cleanup_err; + +- if (dmah) +- return ERR_PTR(-EOPNOTSUPP); +- + if (access & ~RXE_ACCESS_SUPPORTED_MR) { + rxe_err_pd(pd, "access = %#x not supported (%#x)\n", access, + RXE_ACCESS_SUPPORTED_MR); +--- a/rxe_odp.c 2026-08-11 20:26:52.908892000 +0000 ++++ b/rxe_odp.c 2026-08-11 20:18:05.338000000 +0000 +@@ -27,7 +27,7 @@ + start = max_t(u64, ib_umem_start(umem_odp), range->start); + end = min_t(u64, ib_umem_end(umem_odp), range->end); + +- /* update umem_odp->map.pfn_list */ ++ /* update umem_odp->pfn_list */ + ib_umem_odp_unmap_dma_pages(umem_odp, start, end); + + mutex_unlock(&umem_odp->umem_mutex); +@@ -137,7 +137,7 @@ + while (addr < iova + length) { + idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; + +- if (!(umem_odp->map.pfn_list[idx] & HMM_PFN_VALID)) { ++ if (!(umem_odp->pfn_list[idx] & HMM_PFN_VALID)) { + need_fault = true; + break; + } +@@ -201,7 +201,7 @@ + while (length > 0) { + u8 *src, *dest; + +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[idx]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[idx]); + user_va = kmap_local_page(page); + + src = (dir == RXE_TO_MR_OBJ) ? addr : user_va; +@@ -288,7 +288,7 @@ + } + + idx = rxe_odp_iova_to_index(umem_odp, iova); +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[idx]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[idx]); + + va = kmap_local_page(page); + +@@ -347,7 +347,7 @@ + index = rxe_odp_iova_to_index(umem_odp, iova); + page_offset = rxe_odp_iova_to_page_offset(umem_odp, iova); + +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[index]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[index]); + + bytes = min_t(unsigned int, length, + mr_page_size(mr) - page_offset); +@@ -396,7 +396,7 @@ + } + + index = rxe_odp_iova_to_index(umem_odp, iova); +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[index]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[index]); + + va = kmap_local_page(page); + /* Do atomic write after all prior operations have completed */ diff --git a/docker/decoding-server/hw_ci/rxe-ofed/prepare-src.sh b/docker/decoding-server/hw_ci/rxe-ofed/prepare-src.sh new file mode 100755 index 000000000..330da2d59 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/prepare-src.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Stage the pinned upstream rxe (SoftRoCE) driver source with the +# OFED-compat patch applied, ready for a per-host `make` at container setup +# time (see the sibling makefile and README.md for why). +# +# Runs at dev-image build time (dev.Dockerfile); can also be run manually. +# +# prepare-src.sh [kernel-ref] [dest-dir] +set -euo pipefail + +ref=${1:-v6.17} +dest=${2:-/opt/rxe-ofed/src} +here=$(cd "$(dirname "$0")" && pwd) + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +# Sparse partial clone: only the rxe directory's blobs are fetched. +git clone --depth 1 --branch "$ref" --filter=tree:0 --sparse --quiet \ + https://github.com/torvalds/linux.git "$tmp/linux" +git -C "$tmp/linux" sparse-checkout set drivers/infiniband/sw/rxe + +mkdir -p "$dest" +cp "$tmp/linux/drivers/infiniband/sw/rxe/"* "$dest/" +test -f "$dest/rxe.c" # sparse checkout sanity + +# --fuzz=0: any drift between the pinned ref and the patch must fail the +# image build loudly rather than half-apply. +patch -d "$dest" -p1 --fuzz=0 < "$here/ofed-compat.patch" + +cp "$here/rxe_ofed_compat.h" "$here/makefile" "$dest/" +echo "rxe-ofed: staged $ref + ofed-compat.patch at $dest" diff --git a/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h b/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h new file mode 100644 index 000000000..9c1741eab --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * The MLNX/DOCA-OFED compat enum ib_odp_transport_cap_bits ends at + * IB_ODP_SUPPORT_SRQ_RECV (1 << 5); these two bits exist upstream in v6.17 + * (values mirror uapi IB_UVERBS_ODP_SUPPORT_* exactly). Injected via + * -include from the accompanying makefile so the upstream rxe sources need + * no edit for this. + */ +#ifndef RXE_OFED_COMPAT_H +#define RXE_OFED_COMPAT_H + +#define IB_ODP_SUPPORT_FLUSH (1 << 6) +#define IB_ODP_SUPPORT_ATOMIC_WRITE (1 << 7) + +#endif diff --git a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh index b010653ad..cafde80a0 100755 --- a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh +++ b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh @@ -128,6 +128,12 @@ IB_DEVICE="" # auto-detect first Up ConnectX BRIDGE_IP="192.168.0.1" FPGA_IP="192.168.0.2" MTU=4096 +# RDMA ring slot size in bytes (--page-size). Feeds the cpu_roce wire's +# --slot-size, the device_graph ring's QEC_DEVICE_GRAPH_FRAME_SIZE, and +# playback --page-size. device_graph requires the ring (64 slots x this) to +# be a multiple of the host page size, so 64 KiB-page hosts (e.g. GB200 +# -64k kernels) reject the default; the run_fpga geometry check suggests a +# working value. PAGE_SIZE=384 # RX ring depth for both FPGA paths, bounded by the HSB QP's 64 receive WQEs # (the FPGA writes frame rid to slot rid % NUM_SLOTS; more slots than WQEs drops @@ -221,6 +227,13 @@ FPGA-only: fires once per FRAME (one BRAM window), so a shot spans frames-per-shot x spacing; keeps the RX ring from overrunning + --page-size N RDMA ring slot size in bytes (default 384); feeds + the cpu_roce wire's --slot-size, the device_graph + ring's QEC_DEVICE_GRAPH_FRAME_SIZE, and playback + --page-size. device_graph requires the ring + (64 slots x N) to be a multiple of the host page + size; on non-4K-page hosts the geometry check + computes a compliant value --no-verify Skip playback correction verification --help, -h Show this help @@ -254,6 +267,7 @@ while [[ $# -gt 0 ]]; do --fpga-ip) FPGA_IP="$2"; shift ;; --mtu) MTU="$2"; shift ;; --spacing) SPACING="$2"; SPACING_EXPLICIT=true; shift ;; + --page-size) PAGE_SIZE="$2"; shift ;; --wire) WIRE="$2"; shift ;; --dispatch) DISPATCH="$2"; shift ;; --no-verify) VERIFY=false ;; @@ -1003,9 +1017,12 @@ run_fpga() { # with the constraint spelled out. local host_page; host_page=$(getconf PAGESIZE) if (( (DEVICE_GRAPH_NUM_PAGES * PAGE_SIZE) % host_page != 0 )); then + local stride=$(( host_page / HSB_WQE_DEPTH )) + local suggest=$(( (PAGE_SIZE + stride - 1) / stride * stride )) _err "device_graph ring ($DEVICE_GRAPH_NUM_PAGES slots x $PAGE_SIZE B) is not a multiple of this host's page size ($host_page B)." - _err "The HSB frame stride must be a multiple of $(( host_page / HSB_WQE_DEPTH )) B on this host; see the unittests" - _err "hsb_fpga_decoding_server_test.sh (--page-size) for a tunable-geometry run." + _err "Pass --page-size as a multiple of $stride B on this host (smallest >= $PAGE_SIZE: $suggest)." + _err "For reference, on 64 KiB-page GB200 hosts, host dispatch runs at --page-size 512 and" + _err "device_graph at the rounded-up value (1024)." exit 1 fi fi diff --git a/libs/qec/unittests/realtime/app_examples/CMakeLists.txt b/libs/qec/unittests/realtime/app_examples/CMakeLists.txt index 591fb7f35..2504ffd0c 100644 --- a/libs/qec/unittests/realtime/app_examples/CMakeLists.txt +++ b/libs/qec/unittests/realtime/app_examples/CMakeLists.txt @@ -365,6 +365,14 @@ if(TARGET cudaq-qec-realtime-decoding-server-cqr) 2 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) + + # The script exits 77 on transports it cannot exercise (per-decoder ring + # endpoints are udp-only), and TIMEOUT bounds a wedged server well below + # the 1500 s ctest default. + set_tests_properties( + app_examples.surface_code-1-cqr-two-process-test-distance-3 + app_examples.surface_code-1-cqr-two-process-test-distance-3-num-logical-2 + PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 300) endif() endif() diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh b/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh index c09301342..f1c883a37 100644 --- a/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh +++ b/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh @@ -53,6 +53,16 @@ NUM_ROUNDS=$6 DECODER_TYPE=${7:-multi_error_lut} NUM_LOGICAL=${8:-1} +# Device-scoped endpoint args exist only for udp (udp-port.=): over +# cpu_roce the app refuses --num_logical > 1, and the server would then wedge +# in ring 1's blocking rendezvous accept(). Skip with a named reason +# (SKIP_RETURN_CODE 77 in the ctest registration). +if [[ "${QEC_DECODING_SERVER_TRANSPORT:-udp}" != "udp" && "$NUM_LOGICAL" -gt 1 ]]; then + echo "SKIP: per-decoder ring endpoints exercised over udp only (no" \ + "device-scoped ${QEC_DECODING_SERVER_TRANSPORT} channel args)" + exit 77 +fi + export CUDAQ_DEFAULT_SIMULATOR=stim NUM_SHOTS=1000 @@ -86,10 +96,23 @@ fi QEC_DECODING_SERVER_STATS=1 $SERVER_PATH "${SERVER_ARGS[@]}" \ > $SERVER_LOG 2>&1 & SERVER_PID=$! -cleanup() { +# TERM first; escalate to KILL after a 5 s grace. The cpu_roce bridge's +# rendezvous connect() blocks in accept() where the server's shutdown flag +# is never checked, and a wedged server must fail the test fast rather +# than sit out the whole ctest timeout. +stop_server() { kill -TERM $SERVER_PID 2>/dev/null || true + for _ in $(seq 1 50); do + kill -0 $SERVER_PID 2>/dev/null || break + sleep 0.1 + done + if kill -0 $SERVER_PID 2>/dev/null; then + echo "WARNING: decoding_server ignored SIGTERM; sending SIGKILL" + kill -KILL $SERVER_PID 2>/dev/null || true + fi wait $SERVER_PID 2>/dev/null || true } +cleanup() { stop_server; } trap cleanup EXIT # Wait for the READY line and parse the port. @@ -133,8 +156,7 @@ QEC_DECODING_SERVER_PORT=$SERVER_PORT \ |& tee $APP_LOG # [4] Stop the server and collect its dispatch count. -kill -TERM $SERVER_PID -wait $SERVER_PID 2>/dev/null || true +stop_server trap - EXIT num_non_zero_values=$(grep "Number of non-zero values measured :" $APP_LOG | awk -F': ' '{print $2}') diff --git a/libs/qec/unittests/realtime/test_decoding_server.cpp b/libs/qec/unittests/realtime/test_decoding_server.cpp index 4448329d1..f4904fda7 100644 --- a/libs/qec/unittests/realtime/test_decoding_server.cpp +++ b/libs/qec/unittests/realtime/test_decoding_server.cpp @@ -413,6 +413,14 @@ __qpu__ std::int64_t dual_decoding_server_kernel() { } TEST(DecodingServerTwoProcess, TwoProcessHostDispatchDualDecoders) { + // Shared-wire demux is udp-only: a multi-decoder server opens one ring + // per decoder, and on rendezvous transports (cpu_roce) its post-READY + // connect() blocks until every ring is dialed -- this caller dials one + // endpoint (see decoding_server.cpp [3]), so the test would hang, not + // fail. Multi-ring coverage is TwoProcessPerDecoderRings (also udp-only + // today: device-scoped cpu_roce endpoint args do not exist yet). + if (env_or("QEC_DECODING_SERVER_TRANSPORT", "udp") != "udp") + GTEST_SKIP() << "shared-wire dual decoders demux over udp only"; // Two identical 3-bit-identity pymatching decoders (ids 0 and 1) in one // server -- one per logical qubit. const std::string config_path =