From 0b4c2c30301b1cd09765810a895fcf1d414c3c9a Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Mon, 10 Aug 2026 17:17:30 +0000 Subject: [PATCH 1/9] [QEC] Add decoding-server hardware CI (NVQLink lab) + demo --page-size Hardware-in-the-loop CI for the decoding server on machines with real ConnectX NICs, an FPGA syndrome source, and CC >= 9.0 GPUs -- none of which GitHub CI has (its A100/L4 runners make every device_graph / CUDA-graph path structurally unreachable). New: docker/decoding-server/hw_ci/run_hw_ci.sh Host-side runner: takes a cudaqx commit, clones it, builds the dev image locally (layer-cached, never pushed), fail-fasts if the image's baked CUDA-Q mismatches the commit's .cudaq_version pin (--build-base builds the base locally for pre-merge pin bumps), then runs 23 lanes sequentially with a PASS/FAIL/SKIP(reason) summary (exit-77 = SKIP; --strict makes skips fatal). Lanes: the realtime_decoding_demo matrix (4 decoders x udp / two-process cpu_roce / FPGA cpu_roce host dispatch / FPGA device_graph), the two-process cpu_roce ctests, the hsb_fpga decoding-server script over cpu_roce and gpu_roce, the CC>=9.0-only ctests (qldpc-graph, mixed-dispatch, FP8 ONNX), and the gpu_roce QLDPC bridge as a below-the-server cross-check. The Ising TRT bundle is rebuilt from the gated HF model on every run (ising-prepare lane; HF_TOKEN passed only to that lane's exec). Proprietary artifacts (nv-qldpc plugin, cudevice archive) come from a read-only bind mount; missing pieces surface as named SKIPs. CUDA arch is auto-detected (GB200=100, Spark GB10=121). Two supported configurations: DGX Spark with the loopback cable (--no-fpga --roce-pair DEV0,DEV1; the single cable is FPGA XOR loopback) and GB200 with the FPGA cabled (--roce-pair rxe --fpga-device DEV). New: docker/decoding-server/dev.Dockerfile Build+run environment on ghcr.io/nvidia/cudaqx-dev: RDMA userspace, DOCA 3.3.0 gpunetio-dev, Holoscan SDK, TensorRT, cuStabilizer, the Ising exporter's Python env (CPU torch) + hf CLI, and a prebuilt holoscan-sensor-bridge 2.6.0-EA2 -- the build_qec.sh test-time installs moved into cached image layers. cudaq-realtime (HSB=ON) and cudaqx itself are built per-commit by hw_ci/container_build.sh, not baked in. Known gap documented in the README: the base's Mellanox ibverbs-providers has no rxe provider, so SoftRoCE lanes currently SKIP. Demo (docs/sphinx/examples/qec/realtime_decoding_demo): - run_realtime_decoding.sh: new --page-size option (default 384). The fixed 384-byte geometry made the device_graph mode refuse to run on 16K/64K-page kernels -- i.e. on stock GB200 -64k images, the advertised platform; the fail-fast now computes and suggests a compliant value instead of dead-ending. - Fix playback invocation: --hololink was silently ignored by hsb_fpga_syndrome_playback after the #754 rename (it only parses --hsb-ip), so every FPGA run of the example exited with usage. Signed-off-by: Chuck Ketcham --- docker/decoding-server/dev.Dockerfile | 175 +++++ docker/decoding-server/hw_ci/README.md | 98 +++ .../decoding-server/hw_ci/container_build.sh | 160 ++++ docker/decoding-server/hw_ci/run_hw_ci.sh | 698 ++++++++++++++++++ .../run_realtime_decoding.sh | 23 +- 5 files changed, 1151 insertions(+), 3 deletions(-) create mode 100644 docker/decoding-server/dev.Dockerfile create mode 100644 docker/decoding-server/hw_ci/README.md create mode 100755 docker/decoding-server/hw_ci/container_build.sh create mode 100755 docker/decoding-server/hw_ci/run_hw_ci.sh diff --git a/docker/decoding-server/dev.Dockerfile b/docker/decoding-server/dev.Dockerfile new file mode 100644 index 000000000..12e5e1d94 --- /dev/null +++ b/docker/decoding-server/dev.Dockerfile @@ -0,0 +1,175 @@ +# ============================================================================ # +# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # + +# Development image for building AND running the QEC decoding server on +# RDMA/FPGA hardware (NVQLink lab, DGX Spark / GB200). +# +# The base image (ghcr.io/nvidia/cudaqx-dev, built by +# docker/build_env/cudaqx.dev.Dockerfile) already carries the toolchain plus +# CUDA-Q and cudaq-realtime built at the .cudaq_version pin, with a copy of +# that pin at /cudaq_version. This file adds only the environment the +# hardware paths need -- the same packages CI installs at test time in +# .github/actions/build-lib/build_qec.sh, moved into cached image layers: +# +# - RDMA userspace (rdma-core, ibverbs providers incl. SoftRoCE/rxe) +# - DOCA 3.3.0 GPUNetIO dev headers (NOT doca-all: it conflicts with the +# Mellanox OFED preinstalled in the devcontainer base) +# - Holoscan SDK +# - TensorRT dev (for the trt_decoder plugin; arm64 packages exist for +# CUDA 13 only, hence the cu13.0 default base) +# - cuStabilizer (cuquantum-python wheel) +# - the Ising-artifact exporter's Python environment + `hf` CLI, so the +# hardware CI can download the gated model and rebuild the bundle on +# every run +# - a prebuilt holoscan-sensor-bridge 2.6.0-EA2 at /opt/holoscan-sensor-bridge +# +# What is deliberately NOT baked in: cudaq-realtime-with-HSB-tools and cudaqx +# itself -- the hardware CI builds both from the commit under test (see +# docker/decoding-server/hw_ci/container_build.sh). Proprietary artifacts +# (nv-qldpc plugin, cudevice archive) are bind-mounted at run time, never +# baked into a layer. +# +# Build (normally done by hw_ci/run_hw_ci.sh, context = this directory): +# docker build -f docker/decoding-server/dev.Dockerfile \ +# --build-arg base_image=ghcr.io/nvidia/cudaqx-dev: \ +# --build-arg cuda_native_arch=100 \ +# -t cudaqx-decoding-hwci docker/decoding-server + +ARG base_image=ghcr.io/nvidia/cudaqx-dev:latest-arm64-cu13.0 +FROM ${base_image} + +# CUDA architecture the prebaked holoscan-sensor-bridge kernels target: +# 100 = GB200 (sm_100), 121 = DGX Spark GB10 (sm_121). The hardware-CI +# runner auto-detects and passes this. +ARG cuda_native_arch=100 + +# --------------------------------------------------------------------------- +# Build tools + RDMA userspace. +# NOTE: the base image ships Mellanox OFED's rdma-core fork, whose +# ibverbs-providers outranks Ubuntu's and contains ONLY the mlx5 provider -- +# no rxe (SoftRoCE). The apt line below therefore keeps the Mellanox +# package, and `--roce-pair rxe` runs will SKIP with a named reason until a +# rxe provider matching the Mellanox libibverbs ABI is built into the image +# (see the hw_ci README). perftest (ib_write_bw) is for fabric smoke tests. +# The `sudo` binary must exist because the example scripts' network helpers +# invoke it literally (a no-op when already root). +# --------------------------------------------------------------------------- +RUN apt-get update && apt-get install -y --no-install-recommends \ + ninja-build curl pkg-config jq \ + rdma-core ibverbs-providers ibverbs-utils infiniband-diags perftest \ + iproute2 ethtool iputils-ping sudo \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# --------------------------------------------------------------------------- +# DOCA 3.3.0: only the GPUNetIO dev package (mirrors build_qec.sh; doca-all +# conflicts with the base image's preinstalled OFED), plus cuda-nvrtc-dev +# matching the toolkit (hololink_core links CUDA::nvrtc). +# --------------------------------------------------------------------------- +RUN set -e; \ + DOCA_ARCH=$(uname -m); \ + case "$DOCA_ARCH" in aarch64|arm64) DOCA_ARCH="arm64-sbsa" ;; esac; \ + DOCA_REPO="https://linux.mellanox.com/public/repo/doca/3.3.0/ubuntu24.04/$DOCA_ARCH"; \ + curl -fsSL "$DOCA_REPO/GPG-KEY-Mellanox.pub" -o /usr/share/keyrings/GPG-KEY-Mellanox.pub; \ + echo "deb [signed-by=/usr/share/keyrings/GPG-KEY-Mellanox.pub] $DOCA_REPO /" \ + > /etc/apt/sources.list.d/doca.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends libdoca-sdk-gpunetio-dev; \ + CUDA_FULL_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p'); \ + CUDA_VER_DASH=$(echo "$CUDA_FULL_VERSION" | sed 's/\./-/'); \ + apt-get install -y cuda-nvrtc-dev-$CUDA_VER_DASH 2>/dev/null || true; \ + apt-get clean && rm -rf /var/lib/apt/lists/*; \ + test -d /opt/mellanox/doca/include + +# --------------------------------------------------------------------------- +# Holoscan SDK (force-install fallback mirrors build_qec.sh: the package's +# dependency list can miss on the devcontainer base). +# --------------------------------------------------------------------------- +RUN set -e; \ + CUDA_MAJOR_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p'); \ + apt-get update; \ + apt-get install -y --no-install-recommends holoscan-cuda-$CUDA_MAJOR_VERSION || { \ + _hsdk_tmp=$(mktemp -d); \ + (cd "$_hsdk_tmp" && apt-get download holoscan holoscan-cuda-$CUDA_MAJOR_VERSION \ + && dpkg --force-depends -i holoscan*.deb); \ + rm -rf "$_hsdk_tmp"; \ + }; \ + apt-get clean && rm -rf /var/lib/apt/lists/*; \ + test -d /opt/nvidia/holoscan + +# --------------------------------------------------------------------------- +# TensorRT dev, pinned to the toolkit's CUDA flavor (mirrors the arm64 steps +# in .github/workflows/lib_qec.yaml). +# --------------------------------------------------------------------------- +RUN set -e; \ + CUDA_FULL_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p'); \ + apt-get update; \ + apt-cache search tensorrt \ + | awk -v v="$CUDA_FULL_VERSION" '{print "Package: "$1"\nPin: version *+cuda"v"\nPin-Priority: 1001\n"}' \ + > /etc/apt/preferences.d/tensorrt-cuda$CUDA_FULL_VERSION.pref; \ + apt-get install -y tensorrt-dev; \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# --------------------------------------------------------------------------- +# cuStabilizer (pre-bakes .github/actions/build-lib/setup_custabilizer.sh's +# pip install; sourcing that script at build time then becomes a no-op). +# --------------------------------------------------------------------------- +RUN set -e; \ + CUDA_MAJOR_VERSION=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p'); \ + pip install --no-cache-dir "cuquantum-python-cu${CUDA_MAJOR_VERSION}>=26.3.0" + +# --------------------------------------------------------------------------- +# Ising exporter environment: the `hf` CLI plus every Python package +# examples/qec/realtime_decoding_demo/prepare_ising_artifacts.py checks for, +# so the hardware CI downloads the gated model and regenerates the bundle on +# every run. Torch comes from the PyTorch CPU wheel index: since torch 2.7 +# the plain-PyPI Linux wheels (incl. aarch64) are CUDA builds dragging in +# multi-GB nvidia-* wheels, and the exporter's small inference pass +# (32 samples, d=7) does not need the GPU. +# --------------------------------------------------------------------------- +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu \ + && pip install --no-cache-dir \ + "huggingface_hub[cli]" \ + stim ldpc beliefmatching hydra-core omegaconf onnx \ + pymatching safetensors scipy matplotlib numpy \ + && hf version + +# --------------------------------------------------------------------------- +# Prebuilt holoscan-sensor-bridge 2.6.0-EA2 (mirrors build_qec.sh: same +# operator strip, same targets). Both the source and build trees are kept: +# cudaq-realtime's HSB-tools build and cudaqx's HSB-tools build consume them +# via HOLOSCAN_SENSOR_BRIDGE_{SOURCE,BUILD}_DIR. The /workspaces symlink +# satisfies hsb_fpga_decoding_server_test.sh's default HSB_DIR. +# --------------------------------------------------------------------------- +RUN set -e; \ + git clone --depth 1 --branch 2.6.0-EA2 \ + https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git \ + /opt/holoscan-sensor-bridge; \ + cd /opt/holoscan-sensor-bridge; \ + sed -i '/add_subdirectory(audio_packetizer)/d; /add_subdirectory(compute_crc)/d; \ + /add_subdirectory(csi_to_bayer)/d; /add_subdirectory(image_processor)/d; \ + /add_subdirectory(iq_dec)/d; /add_subdirectory(iq_enc)/d; \ + /add_subdirectory(linux_coe_receiver)/d; /add_subdirectory(linux_receiver)/d; \ + /add_subdirectory(packed_format_converter)/d; /add_subdirectory(sub_frame_combiner)/d; \ + /add_subdirectory(udp_transmitter)/d; /add_subdirectory(emulator)/d; \ + /add_subdirectory(sig_gen)/d; /add_subdirectory(sig_viewer)/d' \ + src/hololink/operators/CMakeLists.txt; \ + export CUDA_NATIVE_ARCH=${cuda_native_arch}; \ + cmake -G Ninja -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DHOLOLINK_BUILD_ONLY_NATIVE=OFF \ + -DHOLOLINK_BUILD_PYTHON=OFF \ + -DHOLOLINK_BUILD_TESTS=OFF \ + -DHOLOLINK_BUILD_TOOLS=OFF \ + -DHOLOLINK_BUILD_EXAMPLES=OFF \ + -DHOLOLINK_BUILD_EMULATOR=OFF; \ + cmake --build build --target gpu_roce_transceiver hololink_core; \ + mkdir -p /workspaces; \ + ln -sfn /opt/holoscan-sensor-bridge /workspaces/holoscan-sensor-bridge + +ENV HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=/opt/holoscan-sensor-bridge \ + HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=/opt/holoscan-sensor-bridge/build diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md new file mode 100644 index 000000000..54e9faa86 --- /dev/null +++ b/docker/decoding-server/hw_ci/README.md @@ -0,0 +1,98 @@ +# Decoding-server hardware CI (NVQLink lab) + +CI-like testing of the QEC decoding server on real hardware — ConnectX RDMA +NICs, an FPGA syndrome source, and CC >= 9.0 GPUs — none of which normal +GitHub CI has (its GPU runners are A100/L4, so every `device_graph` / +CUDA-graph path is unreachable there). + +`run_hw_ci.sh` takes a cudaqx commit, clones it, builds the dev image locally +(`../dev.Dockerfile`, layer-cached), builds cudaq-realtime + cudaqx + the +`realtime_decoding_demo` binaries inside the container +(`container_build.sh`), and runs the test lanes. Every lane ends the summary +as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. + +``` +./run_hw_ci.sh --sha --roce-pair rxe --hf-token-file ~/.hf_token +./run_hw_ci.sh --list # show the lane set +``` + +## Lanes + +* **examples tier** — the shipped `examples/qec/realtime_decoding_demo` + driver: all 4 decoders (pymatching, multi_error_lut, nv-qldpc-decoder, + trt_decoder) over `udp` (baseline), `cpu_roce` two-process (RoCE pair), + and the FPGA source (`cpu_roce` host dispatch ×4, `device_graph` + nv-qldpc ×1); plus `ising-prepare`, which downloads the gated Hugging Face + Ising model and rebuilds the TRT artifact bundle **on every run** — a FAIL + there means the HF/download/export path regressed, independent of decoders. +* **extra tier** — hardware tests outside the examples directory: + `DecodingServerTwoProcess` + the `surface_code-1-cqr` two-process app + ctests over `cpu_roce`; `hsb_fpga_decoding_server_test.sh` over `cpu_roce` + and `gpu_roce` (the FPGA control-plane / SIF playback coverage); the + CC >= 9.0-only ctests (`test_realtime_qldpc_graph_decoding`, + mixed-dispatch, FP8 ONNX); and the `gpu_roce` QLDPC bridge as a + below-the-server cross-check. +* **opt-in** (`--include-opt-in`) — the `gpu_roce` predecoder bridge; it + links the experimental `cudaq-realtime-pipeline` library (off by default, + pending a port to the post-PR4770 graph-launch API), so today it SKIPs. + +## One-time host setup + +1. Docker + nvidia-container-toolkit; user in the `docker` group; + `docker login ghcr.io` (to pull `ghcr.io/nvidia/cudaqx-dev`). +2. For SoftRoCE mode (`--roce-pair rxe`): `sudo modprobe rdma_rxe`, persisted + via `echo rdma_rxe | sudo tee /etc/modules-load.d/rdma_rxe.conf`. + **Known gap:** the dev image inherits Mellanox OFED's `ibverbs-providers`, + which ships only the mlx5 userspace provider — no rxe. Until a rxe + provider matching that libibverbs ABI is built into the image, rxe setup + fails its `ibv_devinfo` preflight and the two-process cpu_roce lanes SKIP + with a named reason. Machines with a real loopback-cabled port pair + (`--roce-pair DEV0,DEV1`) are unaffected. +3. FPGA cabled/flashed and reachable (defaults: NIC 192.168.0.1/24, FPGA + 192.168.0.2). Machines whose single cable is wired as a loopback pair + instead run with `--no-fpga --roce-pair DEV0,DEV1`. +4. Proprietary artifacts (any subset; missing pieces => named SKIPs), staged + under `--artifacts-dir` (default `/opt/nvqlink-lab-artifacts`), mounted + read-only at `/artifacts` in the container: + + ``` + decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so # build against the SHA under test + cudevice/libcudaq-qec-realtime-cudevice-proprietary.a + ``` + +5. Hugging Face access for the Ising lanes: request access once to the gated + `nvidia/Ising-Decoder-SurfaceCode-1-Fast` repo, then provide a token via + `--hf-token-file` or `HF_TOKEN`. The token is passed only to the + `ising-prepare` lane's `docker exec` — never baked into an image or + written to a log. No token => the ising/trt lanes SKIP. +6. ~60 GB free disk for image layers and build trees. + +## Per-machine invocations + +```bash +# GB200 #1 (FPGA on one port, no free port pair -> SoftRoCE for two-process): +./run_hw_ci.sh --sha --roce-pair rxe --fpga-device rocep1s0f0 \ + --hf-token-file ~/.hf_token + +# DGX Spark, single cable in loopback mode (port0 <-> port1, no FPGA): +./run_hw_ci.sh --sha --no-fpga --roce-pair rocep1s0f0,rocep1s0f1 + +# DGX Spark, single cable in FPGA mode: +./run_hw_ci.sh --sha --fpga-device rocep1s0f0 +``` + +The CUDA architecture is auto-detected (`--cuda-arch` to override; GB200 = +100, Spark GB10 = 121). On 64 KiB-page kernels (GB200 `-64k`) the ring slot +size defaults to 512 and the `device_graph` lane rounds up to the +host-page-compatible value; `--page-size` overrides. + +## Image / pin lifecycle + +The dev image is built locally on every run and never pushed; the Docker +layer cache makes unchanged builds take seconds. Its base +(`ghcr.io/nvidia/cudaqx-dev`) carries CUDA-Q at the `.cudaq_version` pin — +the runner picks the pin-matched base tag for the commit under test and +fails fast on a mismatch. For pre-merge commits that bump `.cudaq_version` +(no published base yet), `--build-base` builds the base locally from the +commit's own `docker/build_env/cudaqx.dev.Dockerfile` (multi-hour the first +time per pin, cached after). `--refresh-base` re-pulls a moved base tag. diff --git a/docker/decoding-server/hw_ci/container_build.sh b/docker/decoding-server/hw_ci/container_build.sh new file mode 100755 index 000000000..86cc60723 --- /dev/null +++ b/docker/decoding-server/hw_ci/container_build.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# ============================================================================ # +# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# In-container build for the decoding-server hardware CI. Runs inside the +# dev image (docker/decoding-server/dev.Dockerfile) with the cudaqx checkout +# under test mounted at /workspaces/cudaqx and optional proprietary artifacts +# mounted read-only at /artifacts. +# +# This mirrors the CI recipe in .github/actions/build-lib/build_qec.sh with +# the apt/DOCA/Holoscan/HSB steps removed (baked into the image) and the +# hardware-lab deltas applied: +# - CUDA architecture comes from the runner (GB200=100, DGX Spark=121) +# - holoscan-sensor-bridge is the image's prebuilt /opt tree +# - the proprietary cudevice archive and nv-qldpc plugin are picked up +# from /artifacts when present (never required: absent pieces surface +# later as per-lane SKIPs, not build failures) +# - the trt_decoder plugin build is forced ON (TensorRT is in the image; +# a detection regression should fail the configure loudly) +# - the realtime_decoding_demo example binaries are built as well +# +# Usage (normally invoked by run_hw_ci.sh via docker exec): +# container_build.sh --cuda-arch N +set -euo pipefail + +CUDA_ARCH="" +while [[ $# -gt 0 ]]; do + case "$1" in + --cuda-arch) CUDA_ARCH="$2"; shift ;; + *) echo "ERROR: unknown option: $1" >&2; exit 1 ;; + esac + shift +done +[[ -n "$CUDA_ARCH" ]] || { echo "ERROR: --cuda-arch is required" >&2; exit 1; } + +CUDAQX_SRC=/workspaces/cudaqx +CUDAQ_PREFIX=${CUDAQ_INSTALL_PREFIX:-/usr/local/cudaq} +CUDAQX_INSTALL_PREFIX=/usr/local/cudaqx +CUDAQ_REALTIME_ROOT=/tmp/cudaq-realtime +HSB_ROOT=${HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR:-/opt/holoscan-sensor-bridge} +HSB_BUILD=${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR:-$HSB_ROOT/build} +ARTIFACTS_DIR=/artifacts +NV_QLDPC_PLUGIN=$ARTIFACTS_DIR/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so +CUDEVICE_ARCHIVE=$ARTIFACTS_DIR/cudevice/libcudaq-qec-realtime-cudevice-proprietary.a + +cd "$CUDAQX_SRC" +# setup_custabilizer.sh expands $CUSTABILIZER_ROOT unguarded, which is fatal +# under this script's `set -u` (the CI caller runs without -u). +export CUSTABILIZER_ROOT="${CUSTABILIZER_ROOT:-}" +. .github/actions/build-lib/setup_custabilizer.sh # no-op: wheel is baked in +. scripts/cudaq_realtime_cmake_flags.sh # AVX512 workaround, x86-only + +export CUDA_NATIVE_ARCH="$CUDA_ARCH" + +# --------------------------------------------------------------------------- +# cudaq-realtime with HSB tools, from the commit's own .cudaq_version pin +# (produces libcudaq-realtime-bridge-gpu-roce.so, the provider decoding_server +# dlopens for the gpu_roce wire). Mirrors build_qec.sh lines 12-115. +# --------------------------------------------------------------------------- +CUDAQ_REPO=${CUDAQ_REPO:-$(jq -r '.cudaq.repository' .cudaq_version)} +CUDAQ_REF=${CUDAQ_REF:-$(jq -r '.cudaq.ref' .cudaq_version)} +echo "== cudaq-realtime source: ${CUDAQ_REPO}@${CUDAQ_REF}" + +cd /tmp +rm -rf cudaq-realtime-src "$CUDAQ_REALTIME_ROOT" +git clone --filter=blob:none --no-checkout "https://github.com/${CUDAQ_REPO}.git" cudaq-realtime-src +cd cudaq-realtime-src +git sparse-checkout init --cone +git sparse-checkout set realtime cmake +git checkout "$CUDAQ_REF" + +cd realtime +mkdir -p build && cd build +cmake -G Ninja -DCMAKE_INSTALL_PREFIX="$CUDAQ_REALTIME_ROOT" \ + -DCMAKE_CUDA_FLAGS="$(cudaq_realtime_cmake_cuda_flags)" \ + -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCH" \ + -DCUDAQ_REALTIME_ENABLE_HSB_TOOLS=ON \ + -DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR="$HSB_ROOT" \ + -DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR="$HSB_BUILD" \ + .. +ninja +ninja install + +# --------------------------------------------------------------------------- +# Proprietary inputs (optional). +# --------------------------------------------------------------------------- +_prop_archive_flag="" +if [[ -f "$CUDEVICE_ARCHIVE" ]]; then + _prop_archive_flag="-DCUDAQ_QEC_REALTIME_CUDEVICE_PROPRIETARY_ARCHIVE=$CUDEVICE_ARCHIVE" + echo "== cudevice proprietary archive: $CUDEVICE_ARCHIVE" +else + echo "== cudevice proprietary archive: absent (device_graph-dispatch tests will SKIP)" +fi +if [[ -f "$NV_QLDPC_PLUGIN" ]]; then + # Configure-time gate for test_realtime_qldpc_graph_decoding and the + # mixed-dispatch app example. + export QEC_EXTERNAL_DECODERS="$NV_QLDPC_PLUGIN" + echo "== nv-qldpc plugin: $NV_QLDPC_PLUGIN" +else + echo "== nv-qldpc plugin: absent (nv-qldpc lanes will SKIP)" +fi + +# --------------------------------------------------------------------------- +# cudaqx qec. Configured from the TOP-LEVEL CMakeLists (not -S libs/qec) so +# the build tree lands at build/libs/qec/... -- the layout every in-tree +# hardware script hardcodes (hsb_fpga_decoding_server_test.sh & co. default +# to CUDAQX_DIR=/workspaces/cudaqx and resolve binaries + LD paths from +# build/libs/qec/...). Mirrors build_qec.sh lines 120-141 plus the deltas +# listed in the header. +# --------------------------------------------------------------------------- +cd "$CUDAQX_SRC" +cmake -S . -B build \ + -DCUDAQX_ENABLE_LIBS=qec \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc-12 \ + -DCMAKE_CXX_COMPILER=g++-12 \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCH" \ + -DCUDAQ_DIR="$CUDAQ_PREFIX/lib/cmake/cudaq/" \ + -DCUDAQX_INCLUDE_TESTS=ON \ + -DCUDAQX_BINDINGS_PYTHON=ON \ + -DCMAKE_INSTALL_PREFIX="$CUDAQX_INSTALL_PREFIX" \ + -DCUDAQ_REALTIME_ROOT="$CUDAQ_REALTIME_ROOT" \ + -DCUDAQX_QEC_ENABLE_HSB_TOOLS=ON \ + -DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR="$HSB_ROOT" \ + -DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR="$HSB_BUILD" \ + -DCUDAQ_QEC_BUILD_TRT_DECODER=ON \ + $_prop_archive_flag +cmake --build build --target install -j "$(nproc)" + +# The decoding server discovers decoder plugins in +# /lib/decoder-plugins; expose the proprietary one when staged. +if [[ -f "$NV_QLDPC_PLUGIN" ]]; then + mkdir -p "$CUDAQX_INSTALL_PREFIX/lib/decoder-plugins" + ln -sfn "$NV_QLDPC_PLUGIN" \ + "$CUDAQX_INSTALL_PREFIX/lib/decoder-plugins/$(basename "$NV_QLDPC_PLUGIN")" +fi + +# --------------------------------------------------------------------------- +# realtime_decoding_demo example binaries (the examples-tier lanes). The +# arch override is mandatory: the demo's CMakeLists defaults to 80. +# --------------------------------------------------------------------------- +cmake -S docs/sphinx/examples/qec/realtime_decoding_demo -B demo-build -G Ninja \ + -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCH" \ + -DCUDAQ_INSTALL_DIR="$CUDAQ_PREFIX" \ + -DCUDAQX_INSTALL_DIR="$CUDAQX_INSTALL_PREFIX" \ + -DCUDAQ_REALTIME_DIR="$CUDAQ_REALTIME_ROOT" +cmake --build demo-build -j "$(nproc)" + +echo "== container build complete" +echo " cudaqx install : $CUDAQX_INSTALL_PREFIX" +echo " realtime : $CUDAQ_REALTIME_ROOT" +echo " qec build tree : $CUDAQX_SRC/build" +echo " demo binaries : $CUDAQX_SRC/demo-build" diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh new file mode 100755 index 000000000..b0209a678 --- /dev/null +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -0,0 +1,698 @@ +#!/bin/bash +# ============================================================================ # +# Copyright (c) 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# Hardware CI runner for the QEC decoding server (NVQLink lab: DGX Spark / +# GB200 with ConnectX NICs and an FPGA syndrome source). +# +# Given a cudaqx commit, this script -- run on the HOST, not in a container: +# 1. clones cudaqx at that commit into the work dir +# 2. builds the dev image (docker/decoding-server/dev.Dockerfile) locally; +# the Docker layer cache makes unchanged builds near-instant +# 3. fails fast if the image's baked CUDA-Q does not match the commit's +# .cudaq_version pin +# 4. builds cudaq-realtime + cudaqx + the demo binaries inside the +# container (hw_ci/container_build.sh) +# 5. runs the test lanes (see --list) and prints a PASS/FAIL/SKIP summary +# +# Lanes report SKIP (exit 77) when an input is absent -- a proprietary +# artifact, the HF token, the FPGA -- so lost coverage is always visible in +# the summary without failing the run; --strict turns skips into failures. +# +# Proprietary artifacts are staged once on the host (see --artifacts-dir); +# the Ising/TRT model bundle is deliberately NOT staged: the ising-prepare +# lane downloads the gated Hugging Face model and rebuilds the bundle inside +# the container on every run, so that path is continuously validated. +set -uo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +SHA="" +REPO_URL="https://github.com/NVIDIA/cudaqx.git" +WORKDIR="$HOME/.cache/cudaqx-hw-ci" +ARTIFACTS_DIR="/opt/nvqlink-lab-artifacts" +TIER="all" # examples | extra | all +INCLUDE_OPT_IN=false +ONLY_GLOB="" +SKIP_GLOB="" +STRICT=false +LIST_ONLY=false +REFRESH_BASE=false +BASE_IMAGE="" # resolved from the pin unless given +BUILD_BASE=false +CUDA_VERSION="13.0" +CUDA_ARCH="" # auto-detected unless given (Spark=121, GB200=100) +ROCE_PAIR="" # rxe | DEV0,DEV1 ; empty = skip cpu_roce pair lanes +HF_TOKEN_FILE="" +FPGA_DEVICE="" # ConnectX IB device facing the FPGA +BRIDGE_IP="192.168.0.1" +FPGA_IP="192.168.0.2" +PAGE_SIZE="" # default derived from the host page size +KEEP_CONTAINER=false +NO_FPGA=false + +print_usage() { + cat < the dependent lanes SKIP + +Lane selection: + --tier T examples | extra | all (default: all) + --include-opt-in also run the opt-in lanes (see --list) + --only GLOB run only lanes matching GLOB (e.g. 'examples/fpga/*') + --skip GLOB skip lanes matching GLOB + --list print the lane list for the current flags and exit + --strict any SKIP fails the run (full-coverage mode) + +Image: + --base-image IMG override the cudaqx-dev base image + --refresh-base docker build --pull (accept a moved base tag; + invalidates cached layers built on the old base) + --build-base if the pin-matched base tag cannot be pulled, build + it locally from the commit's cudaqx.dev.Dockerfile + (multi-hour CUDA-Q build the first time per pin) + --cuda-version V base image CUDA flavor (default: $CUDA_VERSION) + +Hardware: + --cuda-arch N CUDA architecture (default: auto via nvidia-smi; + DGX Spark GB10=121, GB200=100) + --roce-pair rxe SoftRoCE self-loop for the two-process cpu_roce + lanes (GB200 #1: no free ConnectX port pair), OR + --roce-pair DEV0,DEV1 a real loopback-cabled IB device pair (DGX Spark); + omitted => those lanes SKIP + --fpga-device DEV ConnectX IB device facing the FPGA (default: the + test scripts auto-detect / rocep1s0f0) + --bridge-ip IP server-side NIC IP (default $BRIDGE_IP) + --fpga-ip IP FPGA IP (default $FPGA_IP) + --page-size N RDMA ring slot size (default: 384, or 512 on + 64 KiB-page hosts; the device_graph lane rounds up + to the host-page-compatible value) + --no-fpga skip all FPGA lanes (e.g. Spark cabled in loopback + mode: the single cable is FPGA XOR loopback) + +Misc: + --hf-token-file FILE Hugging Face token for the gated Ising model (or + set HF_TOKEN); absent => ising/trt lanes SKIP + --keep-container leave the container running afterwards (debugging) + --help, -h this help +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --sha) SHA="$2"; shift ;; + --repo) REPO_URL="$2"; shift ;; + --workdir) WORKDIR="$2"; shift ;; + --artifacts-dir) ARTIFACTS_DIR="$2"; shift ;; + --tier) TIER="$2"; shift ;; + --include-opt-in) INCLUDE_OPT_IN=true ;; + --only) ONLY_GLOB="$2"; shift ;; + --skip) SKIP_GLOB="$2"; shift ;; + --list) LIST_ONLY=true ;; + --strict) STRICT=true ;; + --base-image) BASE_IMAGE="$2"; shift ;; + --refresh-base) REFRESH_BASE=true ;; + --build-base) BUILD_BASE=true ;; + --cuda-version) CUDA_VERSION="$2"; shift ;; + --cuda-arch) CUDA_ARCH="$2"; shift ;; + --roce-pair) ROCE_PAIR="$2"; shift ;; + --fpga-device) FPGA_DEVICE="$2"; shift ;; + --bridge-ip) BRIDGE_IP="$2"; shift ;; + --fpga-ip) FPGA_IP="$2"; shift ;; + --page-size) PAGE_SIZE="$2"; shift ;; + --no-fpga) NO_FPGA=true ;; + --hf-token-file) HF_TOKEN_FILE="$2"; shift ;; + --keep-container) KEEP_CONTAINER=true ;; + --help|-h) print_usage; exit 0 ;; + *) echo "ERROR: unknown option: $1" >&2; print_usage >&2; exit 1 ;; + esac + shift +done + +_info() { echo "[hw-ci] $*"; } +_err() { echo "[hw-ci] ERROR: $*" >&2; } +_die() { _err "$*"; exit 1; } + +[[ -n "$SHA" || "$LIST_ONLY" == true ]] || { print_usage >&2; _die "--sha is required"; } +case "$TIER" in examples|extra|all) ;; *) _die "--tier must be examples|extra|all" ;; esac + +# --------------------------------------------------------------------------- +# Lane bookkeeping. Every lane lands in the summary exactly once as +# PASS / FAIL / SKIP(reason). Lanes run strictly sequentially: everything +# here shares one GPU, one FPGA, and one RoCE fabric. +# --------------------------------------------------------------------------- +LANE_NAMES=() +LANE_STATUS=() +LANE_DETAIL=() + +lane_selected() { + local name="$1" + case "$TIER" in + examples) [[ "$name" == examples/* || "$name" == optin/* ]] || return 1 ;; + extra) [[ "$name" == extra/* || "$name" == optin/* ]] || return 1 ;; + esac + [[ "$name" == optin/* && "$INCLUDE_OPT_IN" != true ]] && return 1 + # shellcheck disable=SC2053 + [[ -n "$ONLY_GLOB" && "$name" != $ONLY_GLOB ]] && return 1 + # shellcheck disable=SC2053 + [[ -n "$SKIP_GLOB" && "$name" == $SKIP_GLOB ]] && return 1 + return 0 +} + +record_lane() { # name status detail + LANE_NAMES+=("$1"); LANE_STATUS+=("$2"); LANE_DETAIL+=("$3") +} + +skip_lane() { # name reason + lane_selected "$1" || return 0 + if [[ "$LIST_ONLY" == true ]]; then echo " $1 [would SKIP: $2]"; return 0; fi + _info "SKIP $1: $2" + record_lane "$1" SKIP "$2" +} + +# run_lane NAME CMD -- CMD is a bash command line executed in the container. +# Extra `docker exec` flags (e.g. -e VAR=...) come from LANE_ENV. +LANE_ENV=() +run_lane() { + # Consume LANE_ENV first thing so a deselected/listed lane can never leak + # its env flags (or the HF token) into the next lane that runs. + local env_flags=(${LANE_ENV[@]+"${LANE_ENV[@]}"}) + LANE_ENV=() + local name="$1"; shift + lane_selected "$name" || return 0 + if [[ "$LIST_ONLY" == true ]]; then echo " $name"; return 0; fi + local log="$LOG_DIR/${name//\//_}.log" + _info "LANE $name" + local t0=$SECONDS rc=0 + docker exec ${env_flags[@]+"${env_flags[@]}"} "$CONTAINER" bash -lc "$*" \ + >"$log" 2>&1 || rc=$? + local dt=$((SECONDS - t0)) + if [[ $rc -eq 0 ]]; then + record_lane "$name" PASS "${dt}s" + _info "PASS $name (${dt}s)" + elif [[ $rc -eq 77 ]]; then + local reason + reason=$(grep -Eo 'SKIP[:(].*' "$log" | tail -1) + record_lane "$name" SKIP "${reason:-exit 77 (see $log)}" + _info "SKIP $name: ${reason:-exit 77}" + else + record_lane "$name" FAIL "rc=$rc log: $log" + _err "FAIL $name (rc=$rc) log: $log" + fi +} + +print_summary() { + local pass=0 fail=0 skip=0 i + echo + echo "================ HW-CI SUMMARY (sha ${SHORT_SHA:-?}) ================" + for i in "${!LANE_NAMES[@]}"; do + printf '%-5s %-48s %s\n' "${LANE_STATUS[$i]}" "${LANE_NAMES[$i]}" "${LANE_DETAIL[$i]}" + case "${LANE_STATUS[$i]}" in + PASS) ((pass++)) ;; FAIL) ((fail++)) ;; SKIP) ((skip++)) ;; + esac + done + echo "---------------------------------------------------------------" + echo "$pass passed, $fail failed, $skip skipped" + [[ $fail -gt 0 ]] && return 1 + if [[ "$STRICT" == true && $skip -gt 0 ]]; then + echo "--strict: treating $skip skip(s) as failure" + return 1 + fi + return 0 +} + +# --------------------------------------------------------------------------- +# Host-side setup +# --------------------------------------------------------------------------- +detect_cuda_arch() { + [[ -n "$CUDA_ARCH" ]] && return 0 + local cap + cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d ' .') + [[ -n "$cap" ]] || _die "cannot auto-detect the GPU (nvidia-smi); pass --cuda-arch" + CUDA_ARCH="$cap" + _info "CUDA architecture: sm_$CUDA_ARCH (auto-detected)" +} + +checkout_sha() { + SRC="$WORKDIR/src" + mkdir -p "$WORKDIR" + if [[ ! -d "$SRC/.git" ]]; then + _info "Cloning $REPO_URL" + git clone "$REPO_URL" "$SRC" || _die "clone failed" + fi + if git -C "$SRC" fetch origin "$SHA" 2>/dev/null; then + # FETCH_HEAD is exactly the requested branch/tag/SHA tip; never + # resolve "$SHA" locally here or a branch name would silently hit a + # stale local ref from clone time. + git -C "$SRC" checkout --detach FETCH_HEAD || _die "cannot check out '$SHA'" + else + # Not fetchable by name (e.g. an abbreviated SHA): fetch everything, + # then the ref must resolve -- otherwise fail loudly instead of + # falling back to an arbitrary FETCH_HEAD. + git -C "$SRC" fetch origin || _die "fetch failed" + git -C "$SRC" checkout --detach "$SHA" 2>/dev/null || _die "cannot check out '$SHA'" + fi + SHORT_SHA=$(git -C "$SRC" rev-parse --short=12 HEAD) + _info "Testing cudaqx @ $SHORT_SHA" +} + +platform() { + case "$(uname -m)" in + aarch64|arm64) echo arm64 ;; + x86_64) echo amd64 ;; + *) _die "unsupported platform: $(uname -m)" ;; + esac +} + +resolve_base_image() { + [[ -n "$BASE_IMAGE" ]] && { _info "Base image (override): $BASE_IMAGE"; return 0; } + local plat shortref candidate fallback + plat=$(platform) + shortref=$(jq -r '.cudaq.ref' "$SRC/.cudaq_version" | head -c8) + candidate="ghcr.io/nvidia/cudaqx-dev:${shortref}-${plat}-cu${CUDA_VERSION}" + fallback="ghcr.io/nvidia/cudaqx-dev:latest-${plat}-cu${CUDA_VERSION}" + if docker image inspect "$candidate" >/dev/null 2>&1 || docker pull "$candidate" >/dev/null 2>&1; then + BASE_IMAGE="$candidate" + elif [[ "$BUILD_BASE" == true ]]; then + _info "Pin-matched base $candidate unavailable; building it locally (--build-base)" + docker build -f "$SRC/docker/build_env/cudaqx.dev.Dockerfile" \ + --build-arg base_image="ghcr.io/nvidia/cuda-quantum-devcontainer:${plat}-cu${CUDA_VERSION%%.*}.${CUDA_VERSION#*.}-gcc12-main" \ + --build-arg cuda_version="$CUDA_VERSION" \ + -t "$candidate" "$SRC" || _die "local base build failed" + BASE_IMAGE="$candidate" + elif docker image inspect "$fallback" >/dev/null 2>&1 || docker pull "$fallback" >/dev/null 2>&1; then + _info "Pin-matched base $candidate unavailable; falling back to $fallback" + BASE_IMAGE="$fallback" + else + _die "no usable base image: tried $candidate and $fallback (see --build-base / --base-image)" + fi + _info "Base image: $BASE_IMAGE" +} + +build_image() { + IMAGE="cudaqx-decoding-hwci:$SHORT_SHA" + local pull_flag=() + [[ "$REFRESH_BASE" == true ]] && pull_flag=(--pull) + _info "Building dev image $IMAGE (cached layers reused when unchanged)" + docker build ${pull_flag[@]+"${pull_flag[@]}"} \ + -f "$SRC/docker/decoding-server/dev.Dockerfile" \ + --build-arg base_image="$BASE_IMAGE" \ + --build-arg cuda_native_arch="$CUDA_ARCH" \ + -t "$IMAGE" "$SRC/docker/decoding-server" || _die "docker build failed" +} + +check_cudaq_pin() { + local image_ref src_ref + image_ref=$(docker run --rm "$IMAGE" jq -r '.cudaq.ref' /cudaq_version 2>/dev/null) + src_ref=$(jq -r '.cudaq.ref' "$SRC/.cudaq_version") + if [[ -z "$image_ref" || "$image_ref" == "null" ]]; then + _info "WARNING: image has no /cudaq_version; skipping the stale-pin check" + return 0 + fi + if [[ "$image_ref" != "$src_ref" ]]; then + _die "stale base image: it bakes CUDA-Q $image_ref but the commit pins $src_ref. +Wait for build_dev.yaml to publish the new pin's image, pass --base-image, or use --build-base." + fi + _info "CUDA-Q pin check OK ($src_ref)" +} + +start_container() { + CONTAINER="hwci-$SHORT_SHA" + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + mkdir -p "$WORKDIR/ccache" + local artifacts_mount=() + if [[ -d "$ARTIFACTS_DIR" ]]; then + artifacts_mount=(-v "$ARTIFACTS_DIR:/artifacts:ro") + else + _info "Artifacts dir $ARTIFACTS_DIR absent; proprietary lanes will SKIP" + fi + docker run -d --name "$CONTAINER" \ + --privileged --net=host --gpus all --shm-size=8g \ + --device /dev/infiniband \ + -v "$SRC:/workspaces/cudaqx" \ + -v "$WORKDIR/ccache:/root/.ccache" \ + ${artifacts_mount[@]+"${artifacts_mount[@]}"} \ + "$IMAGE" sleep infinity >/dev/null || _die "docker run failed" +} + +cleanup() { + [[ "$KEEP_CONTAINER" == true ]] && return 0 + if [[ -n "${CONTAINER:-}" ]]; then + teardown_roce_pair 2>/dev/null + docker rm -f "$CONTAINER" >/dev/null 2>&1 + fi +} +trap cleanup EXIT + +in_ctr() { docker exec "$CONTAINER" bash -lc "$*"; } + +# --------------------------------------------------------------------------- +# cpu_roce endpoint pair. +# rxe: SoftRoCE self-loop on a dummy netdev -- both endpoints share +# one rxe device/IP (the pattern documented in the in-tree +# surface_code-1-cqr-two-process-test.sh). Needs rdma_rxe on +# the HOST kernel. +# DEV0,DEV1: real loopback-cabled ConnectX pair (e.g. DGX Spark): +# 10.0.0.1/24 <-> 10.0.0.2/24 with permanent neighbor entries +# (same-host IPs otherwise resolve via lo and RDMA CM times out). +# All netlink/rdma calls run inside the privileged --net=host container, so +# they act on the host netns; objects carry the hwci- prefix for teardown. +# --------------------------------------------------------------------------- +ROCE_READY=false +CH_DEV=""; CH_IP=""; DA_DEV=""; DA_IP="" + +setup_roce_pair() { + [[ -z "$ROCE_PAIR" ]] && return 0 + if [[ "$ROCE_PAIR" == "rxe" ]]; then + if ! grep -qw rdma_rxe /proc/modules; then + if ! sudo -n modprobe rdma_rxe 2>/dev/null; then + _err "rdma_rxe is not loaded; run 'sudo modprobe rdma_rxe' and retry" + return 1 + fi + fi + in_ctr " + ip link add hwci-dummy0 type dummy 2>/dev/null || true + ip addr replace 10.88.0.1/24 dev hwci-dummy0 + ip link set hwci-dummy0 up + rdma link show hwci_rxe0 >/dev/null 2>&1 || \ + rdma link add hwci_rxe0 type rxe netdev hwci-dummy0 + ibv_devinfo -d hwci_rxe0 >/dev/null + " || { _err "SoftRoCE setup failed. Note: if 'rdma link' shows the device but ibv_devinfo cannot open it, the image's Mellanox-OFED ibverbs-providers lacks the rxe userspace provider (a known gap; see the README)"; return 1; } + CH_DEV=hwci_rxe0; CH_IP=10.88.0.1 + DA_DEV=hwci_rxe0; DA_IP=10.88.0.1 + else + local dev0="${ROCE_PAIR%%,*}" dev1="${ROCE_PAIR##*,}" + [[ -n "$dev0" && -n "$dev1" && "$dev0" != "$dev1" ]] \ + || { _err "--roce-pair expects rxe or DEV0,DEV1"; return 1; } + in_ctr " + set -e + net0=\$(ibdev2netdev | awk -v d=$dev0 '\$1==d {print \$5}') + net1=\$(ibdev2netdev | awk -v d=$dev1 '\$1==d {print \$5}') + [ -n \"\$net0\" ] && [ -n \"\$net1\" ] + ip link set \$net0 up; ip link set \$net1 up + ip addr replace 10.0.0.1/24 dev \$net0 + ip addr replace 10.0.0.2/24 dev \$net1 + mac0=\$(cat /sys/class/net/\$net0/address) + mac1=\$(cat /sys/class/net/\$net1/address) + ip neigh replace 10.0.0.2 lladdr \$mac1 nud permanent dev \$net0 + ip neigh replace 10.0.0.1 lladdr \$mac0 nud permanent dev \$net1 + " || { _err "RoCE pair setup failed for $ROCE_PAIR"; return 1; } + CH_DEV="$dev0"; CH_IP=10.0.0.1 + DA_DEV="$dev1"; DA_IP=10.0.0.2 + fi + ROCE_READY=true + _info "cpu_roce pair ready: channel=$CH_DEV/$CH_IP daemon=$DA_DEV/$DA_IP" +} + +teardown_roce_pair() { + [[ "$ROCE_PAIR" == "rxe" && -n "${CONTAINER:-}" ]] || return 0 + in_ctr " + rdma link delete hwci_rxe0 2>/dev/null || true + ip link delete hwci-dummy0 2>/dev/null || true + " || true +} + +roce_env() { # docker exec env flags for the cpu_roce topology + LANE_ENV+=( -e "CUDAQ_CPU_ROCE_TEST_CHANNEL_DEVICE=$CH_DEV" + -e "CUDAQ_CPU_ROCE_TEST_CHANNEL_IP=$CH_IP" + -e "CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE=$DA_DEV" + -e "CUDAQ_CPU_ROCE_TEST_DAEMON_IP=$DA_IP" ) +} + +# --------------------------------------------------------------------------- +# Page-size geometry. cpu_roce host dispatch has no host-page constraint; +# the device_graph ring (64 slots) must total a multiple of the host page +# size, so its value rounds up to the compatible one. +# --------------------------------------------------------------------------- +derive_page_sizes() { + local host_page; host_page=$(getconf PAGESIZE) + if [[ -z "$PAGE_SIZE" ]]; then + PAGE_SIZE=384 + [[ "$host_page" -gt 4096 ]] && PAGE_SIZE=512 + fi + local stride=$(( host_page / 64 )) + PAGE_SIZE_DG=$(( (PAGE_SIZE + stride - 1) / stride * stride )) + _info "page size: $PAGE_SIZE (device_graph ring: $PAGE_SIZE_DG; host page $host_page)" +} + +# --------------------------------------------------------------------------- +# Lane definitions +# --------------------------------------------------------------------------- +DEMO=/workspaces/cudaqx/docs/sphinx/examples/qec/realtime_decoding_demo +DEMO_ARGS="--install-prefix /usr/local/cudaqx --cudaq-prefix /usr/local/cudaq \ +--realtime-lib-dir /tmp/cudaq-realtime --example-build-dir /workspaces/cudaqx/demo-build" +NV_QLDPC_PLUGIN_HOST="$ARTIFACTS_DIR/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so" +CUDEVICE_HOST="$ARTIFACTS_DIR/cudevice/libcudaq-qec-realtime-cudevice-proprietary.a" +NV_QLDPC_PLUGIN_CTR=/artifacts/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so +CUDEVICE_CTR=/artifacts/cudevice/libcudaq-qec-realtime-cudevice-proprietary.a +ISING_BUNDLE=/tmp/ising-bundle +CQ_SRC=/tmp/cudaq-realtime-src + +have_hf_token() { + [[ "$LIST_ONLY" == true ]] && return 0 # --list is host-independent + [[ -n "${HF_TOKEN:-}" ]] && return 0 + [[ -n "$HF_TOKEN_FILE" && -r "$HF_TOKEN_FILE" ]] && return 0 + return 1 +} + +hf_token() { + if [[ -n "${HF_TOKEN:-}" ]]; then echo "$HF_TOKEN"; else cat "$HF_TOKEN_FILE"; fi +} + +ising_ready() { + [[ "$LIST_ONLY" == true ]] && return 0 + in_ctr "test -f $ISING_BUNDLE/metadata.txt" 2>/dev/null +} + +# ctest lanes: `ctest -R` exits 0 when NOTHING matches, which would +# false-PASS a lane whose test was never registered (configure-time gates). +# Count first and convert "no match" into a named SKIP. +ctest_cmd() { # regex reason-when-unregistered + local regex="$1" reason="$2" + echo "cd /workspaces/cudaqx/build && \ +n=\$(ctest -N -R '$regex' 2>/dev/null | sed -n 's/^Total Tests: //p'); \ +if [ \"\${n:-0}\" -eq 0 ]; then echo 'SKIP: $reason'; exit 77; fi; \ +ctest --output-on-failure -R '$regex'" +} + +fpga_dev_flag() { [[ -n "$FPGA_DEVICE" ]] && echo "--device $FPGA_DEVICE"; } + +run_examples_tier() { + local d + + # -- Ising bundle: fresh gated-HF download + export, every run ---------- + if have_hf_token; then + [[ "$LIST_ONLY" == true ]] || LANE_ENV=( -e "HF_TOKEN=$(hf_token)" ) + run_lane "examples/ising-prepare" " + set -e + rm -rf $ISING_BUNDLE + app=\$(find /workspaces/cudaqx/build -name surface_code-4-yaml -type f -perm -u+x | head -1) + [ -n \"\$app\" ] || { echo 'surface_code-4-yaml generator not in the build tree'; exit 1; } + python3 $DEMO/prepare_ising_artifacts.py prepare \ + --app \"\$app\" --artifacts-dir $ISING_BUNDLE --yes" + else + skip_lane "examples/ising-prepare" "no HF token (--hf-token-file or HF_TOKEN)" + fi + + # -- qpu-kernel over udp: the no-hardware baseline ----------------------- + for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do + local name="examples/qpu-kernel/udp/$d" extra="" + case "$d" in + nv-qldpc-decoder) + [[ -f "$NV_QLDPC_PLUGIN_HOST" || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST"; continue; } + extra="--nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0" ;; + trt_decoder) + ising_ready || { skip_lane "$name" "no Ising bundle (ising-prepare failed/skipped)"; continue; } + extra="--ising-artifacts-dir $ISING_BUNDLE" ;; + esac + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source qpu-kernel --decoder $d $DEMO_ARGS $extra" + done + + # -- qpu-kernel over cpu_roce: real RDMA verbs on the RoCE pair ---------- + # No --setup-network: the runner configured the pair itself (the demo's + # helper resolves ports via ibdev2netdev, which cannot see rxe devices). + for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do + local name="examples/qpu-kernel/cpu_roce/$d" extra="" + [[ "$ROCE_READY" == true || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "no cpu_roce pair (--roce-pair not set / setup failed)"; continue; } + case "$d" in + nv-qldpc-decoder) + [[ -f "$NV_QLDPC_PLUGIN_HOST" || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST"; continue; } + extra="--nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0" ;; + trt_decoder) + ising_ready || { skip_lane "$name" "no Ising bundle (ising-prepare failed/skipped)"; continue; } + extra="--ising-artifacts-dir $ISING_BUNDLE" ;; + esac + roce_env + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source qpu-kernel --wire cpu_roce --decoder $d $DEMO_ARGS $extra" + done + + # -- FPGA source, cpu_roce wire, host dispatch --------------------------- + for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do + local name="examples/fpga/cpu_roce/$d" extra="" + [[ "$NO_FPGA" == true ]] && { skip_lane "$name" "--no-fpga"; continue; } + case "$d" in + nv-qldpc-decoder) + [[ -f "$NV_QLDPC_PLUGIN_HOST" || "$LIST_ONLY" == true ]] \ + || { skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST"; continue; } + extra="--nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0" ;; + trt_decoder) + ising_ready || { skip_lane "$name" "no Ising bundle (ising-prepare failed/skipped)"; continue; } + extra="--ising-artifacts-dir $ISING_BUNDLE" ;; + esac + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source fpga --dispatch host --decoder $d \ + --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE $DEMO_ARGS $extra" + done + + # -- FPGA source, device_graph dispatch (nv-qldpc only) ------------------ + local name="examples/fpga/device-graph/nv-qldpc-decoder" + if [[ "$NO_FPGA" == true ]]; then + skip_lane "$name" "--no-fpga" + elif [[ ! -f "$NV_QLDPC_PLUGIN_HOST" && "$LIST_ONLY" != true ]]; then + skip_lane "$name" "missing $NV_QLDPC_PLUGIN_HOST" + elif [[ ! -f "$CUDEVICE_HOST" && "$LIST_ONLY" != true ]]; then + skip_lane "$name" "missing $CUDEVICE_HOST (device_graph dispatch not built)" + else + run_lane "$name" "bash $DEMO/run_realtime_decoding.sh --source fpga --decoder nv-qldpc-decoder \ + --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE_DG --nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --gpu 0 $DEMO_ARGS" + fi +} + +run_extra_tier() { + local hsb=/workspaces/cudaqx/libs/qec/unittests/utils/hsb_fpga_decoding_server_test.sh + + # -- two-process device_call channel over real RDMA verbs ---------------- + if [[ "$ROCE_READY" == true || "$LIST_ONLY" == true ]]; then + LANE_ENV=( -e "QEC_DECODING_SERVER_TRANSPORT=cpu_roce" ); roce_env + run_lane "extra/ctest/two-process-cpu-roce" \ + "$(ctest_cmd 'DecodingServerTwoProcess' 'DecodingServerTwoProcess tests not registered')" + LANE_ENV=( -e "QEC_DECODING_SERVER_TRANSPORT=cpu_roce" ); roce_env + run_lane "extra/ctest/app-two-process-cpu-roce" \ + "$(ctest_cmd 'app_examples.surface_code-1-cqr-two-process' 'two-process app tests not registered')" + else + skip_lane "extra/ctest/two-process-cpu-roce" "no cpu_roce pair (--roce-pair not set / setup failed)" + skip_lane "extra/ctest/app-two-process-cpu-roce" "no cpu_roce pair (--roce-pair not set / setup failed)" + fi + + # -- decoding_server over the FPGA (HSB control plane + SIF playback) ---- + if [[ "$NO_FPGA" == true ]]; then + skip_lane "extra/hsb-fpga-server/cpu_roce" "--no-fpga" + skip_lane "extra/hsb-fpga-server/gpu_roce" "--no-fpga" + else + run_lane "extra/hsb-fpga-server/cpu_roce" \ + "bash $hsb --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE --cuda-quantum-dir $CQ_SRC" + if [[ ( -f "$NV_QLDPC_PLUGIN_HOST" && -f "$CUDEVICE_HOST" ) || "$LIST_ONLY" == true ]]; then + run_lane "extra/hsb-fpga-server/gpu_roce" \ + "bash $hsb --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE_DG --transport gpu_roce --decoder nv-qldpc-decoder \ + --nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR --cuda-quantum-dir $CQ_SRC" + elif [[ ! -f "$NV_QLDPC_PLUGIN_HOST" ]]; then + skip_lane "extra/hsb-fpga-server/gpu_roce" "missing $NV_QLDPC_PLUGIN_HOST" + else + skip_lane "extra/hsb-fpga-server/gpu_roce" "missing $CUDEVICE_HOST (device_graph dispatch not built)" + fi + fi + + # -- GB200-class GPU ctests unreachable in normal CI ---------------------- + run_lane "extra/ctest/qldpc-graph" \ + "$(ctest_cmd '^test_realtime_qldpc_graph_decoding$' 'not registered (needs nv-qldpc plugin + cudevice archive at configure)')" + run_lane "extra/ctest/mixed-dispatch" \ + "$(ctest_cmd 'app_examples.surface_code-4-yaml-mixed-dispatch' 'not registered (needs cudevice archive at configure)')" + run_lane "extra/ctest/ai-decoder-fp8" \ + "$(ctest_cmd 'test_ai_decoder_quantized_onnx' 'not registered')" + + # -- gpu_roce bridge cross-check (same data plane, no server layers) ------ + local gbridge=/workspaces/cudaqx/libs/qec/unittests/utils/gpu_roce_qldpc_graph_decoder_test.sh + if [[ "$NO_FPGA" == true ]]; then + skip_lane "extra/gpu-roce-qldpc-bridge" "--no-fpga" + elif [[ ( ! -f "$NV_QLDPC_PLUGIN_HOST" || ! -f "$CUDEVICE_HOST" ) && "$LIST_ONLY" != true ]]; then + skip_lane "extra/gpu-roce-qldpc-bridge" "missing nv-qldpc plugin and/or cudevice archive" + else + run_lane "extra/gpu-roce-qldpc-bridge" " + set -e + bridge=\$(find /workspaces/cudaqx/build -name gpu_roce_qldpc_graph_decoder_bridge -type f | head -1) + [ -n \"\$bridge\" ] || { echo 'SKIP: bridge executable not built'; exit 77; } + bash $gbridge --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ + --page-size $PAGE_SIZE_DG \ + --cuda-qx-dir /workspaces/cudaqx --cuda-quantum-dir $CQ_SRC \ + --hsb-dir /opt/holoscan-sensor-bridge \ + --proprietary-archive $CUDEVICE_CTR --nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR" + fi +} + +run_optin_tier() { + # The predecoder bridge links the experimental cudaq-realtime-pipeline + # library (CUDAQX_QEC_ENABLE_REALTIME_PIPELINE, OFF by default and + # pending a port to the post-PR4770 graph-launch API), so today this + # lane documents the coverage gap rather than exercising it. + if [[ "$NO_FPGA" == true ]]; then + skip_lane "optin/gpu-roce-predecoder" "--no-fpga" + return 0 + fi + run_lane "optin/gpu-roce-predecoder" " + bridge=\$(find /workspaces/cudaqx/build -name gpu_roce_predecoder_bridge -type f | head -1) + [ -n \"\$bridge\" ] || { echo 'SKIP: bridge not built (experimental realtime_pipeline)'; exit 77; } + bash /workspaces/cudaqx/libs/qec/unittests/realtime/gpu_roce_predecoder_test.sh \ + --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP" +} + +run_all_lanes() { + run_examples_tier + run_extra_tier + run_optin_tier +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +if [[ "$LIST_ONLY" == true ]]; then + ROCE_READY=true # listing shows the full lane set, not this host's skips + PAGE_SIZE=${PAGE_SIZE:-384}; PAGE_SIZE_DG=$PAGE_SIZE + echo "Lanes for --tier $TIER$([[ "$INCLUDE_OPT_IN" == true ]] && echo ' --include-opt-in'):" + run_all_lanes + exit 0 +fi + +detect_cuda_arch +derive_page_sizes +checkout_sha +LOG_DIR="$WORKDIR/logs/$SHORT_SHA" +mkdir -p "$LOG_DIR" +resolve_base_image +build_image +check_cudaq_pin +start_container +setup_roce_pair || _info "continuing without a cpu_roce pair" + +_info "Building cudaq-realtime + cudaqx + demo binaries in the container" +build_log="$LOG_DIR/container_build.log" +if ! in_ctr "bash /workspaces/cudaqx/docker/decoding-server/hw_ci/container_build.sh \ + --cuda-arch $CUDA_ARCH" >"$build_log" 2>&1; then + tail -40 "$build_log" >&2 + _die "container build failed; full log: $build_log" +fi +_info "Container build complete" + +run_all_lanes +print_summary diff --git a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh index 1381185ad..cafde80a0 100755 --- a/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh +++ b/docs/sphinx/examples/qec/realtime_decoding_demo/run_realtime_decoding.sh @@ -128,6 +128,12 @@ IB_DEVICE="" # auto-detect first Up ConnectX BRIDGE_IP="192.168.0.1" FPGA_IP="192.168.0.2" MTU=4096 +# RDMA ring slot size in bytes (--page-size). Feeds the cpu_roce wire's +# --slot-size, the device_graph ring's QEC_DEVICE_GRAPH_FRAME_SIZE, and +# playback --page-size. device_graph requires the ring (64 slots x this) to +# be a multiple of the host page size, so 64 KiB-page hosts (e.g. GB200 +# -64k kernels) reject the default; the run_fpga geometry check suggests a +# working value. PAGE_SIZE=384 # RX ring depth for both FPGA paths, bounded by the HSB QP's 64 receive WQEs # (the FPGA writes frame rid to slot rid % NUM_SLOTS; more slots than WQEs drops @@ -221,6 +227,13 @@ FPGA-only: fires once per FRAME (one BRAM window), so a shot spans frames-per-shot x spacing; keeps the RX ring from overrunning + --page-size N RDMA ring slot size in bytes (default 384); feeds + the cpu_roce wire's --slot-size, the device_graph + ring's QEC_DEVICE_GRAPH_FRAME_SIZE, and playback + --page-size. device_graph requires the ring + (64 slots x N) to be a multiple of the host page + size; on non-4K-page hosts the geometry check + computes a compliant value --no-verify Skip playback correction verification --help, -h Show this help @@ -254,6 +267,7 @@ while [[ $# -gt 0 ]]; do --fpga-ip) FPGA_IP="$2"; shift ;; --mtu) MTU="$2"; shift ;; --spacing) SPACING="$2"; SPACING_EXPLICIT=true; shift ;; + --page-size) PAGE_SIZE="$2"; shift ;; --wire) WIRE="$2"; shift ;; --dispatch) DISPATCH="$2"; shift ;; --no-verify) VERIFY=false ;; @@ -1003,9 +1017,12 @@ run_fpga() { # with the constraint spelled out. local host_page; host_page=$(getconf PAGESIZE) if (( (DEVICE_GRAPH_NUM_PAGES * PAGE_SIZE) % host_page != 0 )); then + local stride=$(( host_page / HSB_WQE_DEPTH )) + local suggest=$(( (PAGE_SIZE + stride - 1) / stride * stride )) _err "device_graph ring ($DEVICE_GRAPH_NUM_PAGES slots x $PAGE_SIZE B) is not a multiple of this host's page size ($host_page B)." - _err "The HSB frame stride must be a multiple of $(( host_page / HSB_WQE_DEPTH )) B on this host; see the unittests" - _err "hsb_fpga_decoding_server_test.sh (--page-size) for a tunable-geometry run." + _err "Pass --page-size as a multiple of $stride B on this host (smallest >= $PAGE_SIZE: $suggest)." + _err "For reference, on 64 KiB-page GB200 hosts, host dispatch runs at --page-size 512 and" + _err "device_graph at the rounded-up value (1024)." exit 1 fi fi @@ -1023,7 +1040,7 @@ run_fpga() { # device_graph ring is DEVICE_GRAPH_NUM_PAGES. local pb_pages="$NUM_SLOTS" if [[ "$DISPATCH" == "device_graph" ]]; then pb_pages="$DEVICE_GRAPH_NUM_PAGES"; fi - local args=( --hololink "$FPGA_IP" --per-round --config "$CONFIG_FILE" + local args=( --hsb-ip "$FPGA_IP" --per-round --config "$CONFIG_FILE" --syndromes "$SYNDROMES_FILE" --qp-number "$SERVER_QP" --rkey "$SERVER_RKEY" --buffer-addr "$SERVER_ADDR" --page-size "$PAGE_SIZE" --num-pages "$pb_pages" ) $VERIFY && args+=(--verify) From c7c740d6b43422007ca97d8254a639d147f21ecf Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Mon, 10 Aug 2026 19:12:44 -0500 Subject: [PATCH 2/9] [QEC] Fix decoding-server hw CI issues found in DGX Spark host validation Host-side validation of the hardware CI runner on a DGX Spark (cu13.0 dev image), run in BOTH cable configurations -- ConnectX loopback pair and FPGA -- with proprietary artifacts and an HF token staged, surfaced issues in the runner, the dev image, the container build, and several never-before-executed test paths (cpu_roce two-process, gpu_roce/ device_graph GPU dispatch, ising/trt). Final state: loopback config 13 passed / 0 failed / 9 skipped (named); FPGA config 14 passed / 7 skipped (named) / 1 known FAIL (see below). Runner (docker/decoding-server/hw_ci/run_hw_ci.sh): - Resolve ibdev -> netdev via host sysfs during pair setup. The old `[ -n ] && [ -n ]` guard never tripped under set -e (short-circuited && failures are exempt), so setup drove `ip` with empty netdev names. - Re-assert the RoCE pair addresses and wait for the IPv4-mapped RoCE GIDs before every cpu_roce lane: NetworkManager-managed ports drop statically added addresses on NM's DHCP retry timer, which killed lanes minutes after a successful setup ("no IPv4-mapped RoCEv2 GID found"). README documents the permanent nmcli fix. - Make --repo authoritative on every run (remote set-url before fetch); a stale origin otherwise pinned all runs to the first-ever --repo. - ctest lanes run with --timeout 900 so a wedged test is bounded well below ctest's 1500 s per-test default. - --hf-token-prompt reads the Hugging Face token from the terminal (hidden input, never written to disk) for runs from shared/public accounts, and tokenless runs fall back to a pre-built Ising bundle staged at /ising-bundle when present -- the trt lanes then still run, and ising-prepare SKIPs with a reason noting the HF download/export path was not exercised. Dev image (docker/decoding-server/dev.Dockerfile): - Install mlnx-ofed-kernel-utils (userspace only, from the DOCA repo): it provides ibdev2netdev, which four in-tree --setup-network helpers shell out to. Modern mlnx-tools no longer ships the tool, and the cudaqx-dev base image never had it. - Install patchelf: libs/qec's add_target_libs_to_wheel patches the rpath of staged external decoder plugins at configure time and fails (with an empty error) when patchelf is absent. - Switch torch to the cu130 CUDA build: the pinned Ising-Decoding exporter is GPU-only (its local_run.sh preflights torch.cuda.is_available()), so the previous CPU-torch choice failed the ising-prepare lane before inference. cu130 matches the image's toolkit and runs on GB200 (sm_100) and Spark GB10 (sm_121). - Build holoscan-sensor-bridge with DOCA_SEND_BLUE_FLAME=0: in-tree mlx5 drivers (e.g. the Spark's -nvidia kernel, no OFED) reject BlueFlame UAR allocation, killing every gpu_roce lane at transceiver start ("Failed to create UAR: DOCA Driver call failure"); NONCACHE doorbells are functionally equivalent for the CI lanes. Verified with a minimal doca_uar_create reproducer: BLUEFLAME fails and NONCACHE succeeds on both rdma-core 2601 and 2510, so this is kernel-driver behavior, not userspace version skew. Runner: the container now runs with --ulimit memlock=-1:-1 (matching the long-lived dev container; --privileged does not raise ulimits). Container build (docker/decoding-server/hw_ci/container_build.sh): - Hand cmake a writable copy of the nv-qldpc plugin: /artifacts is mounted read-only and the external-decoder install patches the plugin's rpath in place. The decoder-plugins symlink then serves the patched copy. Tests (first cpu_roce execution of the two-process suites): - DecodingServerTwoProcess.TwoProcessHostDispatchDualDecoders: add the udp-only skip guard its sibling tests already have. Shared-wire demux cannot work on rendezvous transports: a multi-decoder server opens one ring per decoder and blocks in each ring's connect(), while this caller dials a single endpoint -- the test hung to the ctest timeout. - surface_code-1-cqr-two-process-test.sh: named SKIP (exit 77) for NUM_LOGICAL > 1 off udp (device-scoped endpoint args exist only for udp), and TERM -> KILL escalation in server shutdown: decoding_server ignores SIGTERM while blocked in a rendezvous accept(), which otherwise wedges the script's `wait` until the ctest timeout. That server-side SIGTERM gap is a real cpu_roce robustness bug worth a separate fix in the bridge/server bring-up. - Register SKIP_RETURN_CODE 77 and TIMEOUT 300 for both two-process app tests. Validated end-to-end on the Spark, both configurations: - Loopback: all four decoders (pymatching, multi_error_lut, nv-qldpc-decoder, trt_decoder) PASS over udp and cpu_roce with full dispatch/residual-error criteria; the two-process ctest lanes PASS with the multi-decoder cases skipping by name. - FPGA: all four decoders PASS on the FPGA source (SIF playback, BRAM-verified), device_graph GPU dispatch PASSes (first run on GB10), and hsb-fpga-server PASSes on both cpu_roce and gpu_roce. One known FAIL remains: extra/gpu-roce-qldpc-bridge trips its ILA-capture verification (a pre-arm reading of 194/500 samples suggests stale ILA counter state between lanes) while the product paths it cross-checks PASS at the server level -- tracked separately. - ising-prepare PASSes (gated download + on-GPU export, 8 s); the trt lanes PASS both from a fresh bundle and from a staged tokenless bundle; the qldpc-graph and mixed-dispatch ctests register and PASS with the proprietary artifacts staged. Signed-off-by: Chuck Ketcham --- docker/decoding-server/dev.Dockerfile | 32 +++++- docker/decoding-server/hw_ci/README.md | 29 ++++- .../decoding-server/hw_ci/container_build.sh | 8 +- docker/decoding-server/hw_ci/run_hw_ci.sh | 104 +++++++++++++++--- .../realtime/app_examples/CMakeLists.txt | 8 ++ .../surface_code-1-cqr-two-process-test.sh | 28 ++++- .../realtime/test_decoding_server.cpp | 8 ++ 7 files changed, 188 insertions(+), 29 deletions(-) diff --git a/docker/decoding-server/dev.Dockerfile b/docker/decoding-server/dev.Dockerfile index 12e5e1d94..43c4f3ceb 100644 --- a/docker/decoding-server/dev.Dockerfile +++ b/docker/decoding-server/dev.Dockerfile @@ -126,12 +126,13 @@ RUN set -e; \ # Ising exporter environment: the `hf` CLI plus every Python package # examples/qec/realtime_decoding_demo/prepare_ising_artifacts.py checks for, # so the hardware CI downloads the gated model and regenerates the bundle on -# every run. Torch comes from the PyTorch CPU wheel index: since torch 2.7 -# the plain-PyPI Linux wheels (incl. aarch64) are CUDA builds dragging in -# multi-GB nvidia-* wheels, and the exporter's small inference pass -# (32 samples, d=7) does not need the GPU. +# every run. Torch must be a CUDA build: the pinned Ising-Decoding exporter +# is GPU-only (its local_run.sh preflights torch.cuda.is_available(), so CPU +# torch fails the lane before inference starts). The cu130 index matches +# the image's toolkit and covers GB200 (sm_100) and Spark GB10 (sm_121); +# the multi-GB nvidia-* dependency wheels are the accepted cost. # --------------------------------------------------------------------------- -RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu \ +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu130 \ && pip install --no-cache-dir \ "huggingface_hub[cli]" \ stim ldpc beliefmatching hydra-core omegaconf onnx \ @@ -158,6 +159,12 @@ RUN set -e; \ /add_subdirectory(udp_transmitter)/d; /add_subdirectory(emulator)/d; \ /add_subdirectory(sig_gen)/d; /add_subdirectory(sig_viewer)/d' \ src/hololink/operators/CMakeLists.txt; \ + # In-tree mlx5 drivers (e.g. the DGX Spark's -nvidia kernel, no OFED) + # reject BlueFlame UAR allocation -- doca_uar_create fails and every + # gpu_roce lane dies at transceiver start. NONCACHE doorbells are + # functionally equivalent for the CI lanes. + sed -i 's/#define DOCA_SEND_BLUE_FLAME 1/#define DOCA_SEND_BLUE_FLAME 0/' \ + src/hololink/operators/gpu_roce_transceiver/gpu_roce_transceiver_common.hpp; \ export CUDA_NATIVE_ARCH=${cuda_native_arch}; \ cmake -G Ninja -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ @@ -171,5 +178,20 @@ RUN set -e; \ mkdir -p /workspaces; \ ln -sfn /opt/holoscan-sensor-bridge /workspaces/holoscan-sensor-bridge +# --------------------------------------------------------------------------- +# Tools the base image lacks: +# - ibdev2netdev: the in-tree --setup-network helpers shell out to it to map +# IB devices to netdevs. It lives in mlnx-ofed-kernel-utils (from the +# DOCA repo configured above) -- NOT in modern mlnx-tools, which dropped +# it. Userspace deps only; no DKMS/kernel modules ride along with +# --no-install-recommends. +# - patchelf: libs/qec's add_target_libs_to_wheel patches the rpath of +# staged external decoder plugins at configure time. +# --------------------------------------------------------------------------- +RUN apt-get update && apt-get install -y --no-install-recommends \ + mlnx-ofed-kernel-utils patchelf \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && test -x /usr/sbin/ibdev2netdev && command -v patchelf + ENV HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=/opt/holoscan-sensor-bridge \ HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=/opt/holoscan-sensor-bridge/build diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md index 54e9faa86..e02d0e396 100644 --- a/docker/decoding-server/hw_ci/README.md +++ b/docker/decoding-server/hw_ci/README.md @@ -58,14 +58,30 @@ as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. ``` decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so # build against the SHA under test cudevice/libcudaq-qec-realtime-cudevice-proprietary.a + ising-bundle/ # optional: pre-built Ising/TRT + # bundle for tokenless machines ``` 5. Hugging Face access for the Ising lanes: request access once to the gated `nvidia/Ising-Decoder-SurfaceCode-1-Fast` repo, then provide a token via - `--hf-token-file` or `HF_TOKEN`. The token is passed only to the - `ising-prepare` lane's `docker exec` — never baked into an image or - written to a log. No token => the ising/trt lanes SKIP. + `--hf-token-file`, `HF_TOKEN`, or — on shared/public accounts where the + token must not touch disk — `--hf-token-prompt` (interactive, hidden + input). The token is passed only to the `ising-prepare` lane's + `docker exec` — never baked into an image or written to a log. + Without a token the trt lanes fall back to a pre-built bundle at + `/ising-bundle/` when one is staged (generate it once on + a token-holding machine with `prepare_ising_artifacts.py prepare + --app --artifacts-dir ` and copy the + directory over); `ising-prepare` then SKIPs with a reason noting the HF + download/export path was not exercised. No token and no staged bundle + => the ising/trt lanes SKIP. 6. ~60 GB free disk for image layers and build trees. +7. NetworkManager-managed ports (the DGX OS default) silently drop the + runner's statically assigned `10.0.0.x` pair addresses on NM's DHCP + retry timer. The runner re-asserts the addresses and waits for the + IPv4-mapped RoCE GIDs before every cpu_roce lane, which is normally + enough; the permanent fix is to mark the pair unmanaged: + `sudo nmcli device set managed no` (once per port). ## Per-machine invocations @@ -81,6 +97,13 @@ as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. ./run_hw_ci.sh --sha --fpga-device rocep1s0f0 ``` +`--repo` also accepts a local clone path (avoids pushing while iterating). +If that clone is owned by another user — e.g. it was created from a +devcontainer running as root — git refuses to serve it ("dubious +ownership"); allow it once with +`git config --global --add safe.directory ` and the same for +`/.git` (the clone accesses the repo by its `.git` path). + The CUDA architecture is auto-detected (`--cuda-arch` to override; GB200 = 100, Spark GB10 = 121). On 64 KiB-page kernels (GB200 `-64k`) the ring slot size defaults to 512 and the `device_graph` lane rounds up to the diff --git a/docker/decoding-server/hw_ci/container_build.sh b/docker/decoding-server/hw_ci/container_build.sh index 86cc60723..2b6e95016 100755 --- a/docker/decoding-server/hw_ci/container_build.sh +++ b/docker/decoding-server/hw_ci/container_build.sh @@ -97,10 +97,16 @@ else echo "== cudevice proprietary archive: absent (device_graph-dispatch tests will SKIP)" fi if [[ -f "$NV_QLDPC_PLUGIN" ]]; then + # cudaqx's external-decoder install patches the plugin's rpath IN PLACE + # and /artifacts is mounted read-only -- hand the build a writable copy + # (the decoder-plugins symlink below then serves the patched copy too). + mkdir -p /tmp/hwci-artifacts/decoder-plugins + cp -f "$NV_QLDPC_PLUGIN" /tmp/hwci-artifacts/decoder-plugins/ + NV_QLDPC_PLUGIN="/tmp/hwci-artifacts/decoder-plugins/$(basename "$NV_QLDPC_PLUGIN")" # Configure-time gate for test_realtime_qldpc_graph_decoding and the # mixed-dispatch app example. export QEC_EXTERNAL_DECODERS="$NV_QLDPC_PLUGIN" - echo "== nv-qldpc plugin: $NV_QLDPC_PLUGIN" + echo "== nv-qldpc plugin: $NV_QLDPC_PLUGIN (writable copy from $ARTIFACTS_DIR)" else echo "== nv-qldpc plugin: absent (nv-qldpc lanes will SKIP)" fi diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh index b0209a678..1d146ae5f 100755 --- a/docker/decoding-server/hw_ci/run_hw_ci.sh +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -52,6 +52,7 @@ CUDA_VERSION="13.0" CUDA_ARCH="" # auto-detected unless given (Spark=121, GB200=100) ROCE_PAIR="" # rxe | DEV0,DEV1 ; empty = skip cpu_roce pair lanes HF_TOKEN_FILE="" +HF_TOKEN_PROMPT=false FPGA_DEVICE="" # ConnectX IB device facing the FPGA BRIDGE_IP="192.168.0.1" FPGA_IP="192.168.0.2" @@ -108,7 +109,10 @@ Hardware: Misc: --hf-token-file FILE Hugging Face token for the gated Ising model (or - set HF_TOKEN); absent => ising/trt lanes SKIP + set HF_TOKEN); without a token the trt lanes fall + back to a staged bundle (see --artifacts-dir) or SKIP + --hf-token-prompt read the token from the terminal instead (nothing + written to disk; for shared/public accounts) --keep-container leave the container running afterwards (debugging) --help, -h this help EOF @@ -138,6 +142,7 @@ while [[ $# -gt 0 ]]; do --page-size) PAGE_SIZE="$2"; shift ;; --no-fpga) NO_FPGA=true ;; --hf-token-file) HF_TOKEN_FILE="$2"; shift ;; + --hf-token-prompt) HF_TOKEN_PROMPT=true ;; --keep-container) KEEP_CONTAINER=true ;; --help|-h) print_usage; exit 0 ;; *) echo "ERROR: unknown option: $1" >&2; print_usage >&2; exit 1 ;; @@ -152,6 +157,16 @@ _die() { _err "$*"; exit 1; } [[ -n "$SHA" || "$LIST_ONLY" == true ]] || { print_usage >&2; _die "--sha is required"; } case "$TIER" in examples|extra|all) ;; *) _die "--tier must be examples|extra|all" ;; esac +# Interactive token entry: the token lives only in this process's memory (and +# the ising-prepare lane's docker exec env), never on disk -- for runs from +# shared/public accounts. +if [[ "$HF_TOKEN_PROMPT" == true && "$LIST_ONLY" != true ]]; then + [[ -r /dev/tty ]] || _die "--hf-token-prompt needs a terminal (use --hf-token-file or HF_TOKEN otherwise)" + read -rs -p "Hugging Face token (input hidden): " HF_TOKEN < /dev/tty; echo + [[ -n "$HF_TOKEN" ]] || _die "--hf-token-prompt: empty token" + export HF_TOKEN +fi + # --------------------------------------------------------------------------- # Lane bookkeeping. Every lane lands in the summary exactly once as # PASS / FAIL / SKIP(reason). Lanes run strictly sequentially: everything @@ -256,6 +271,10 @@ checkout_sha() { _info "Cloning $REPO_URL" git clone "$REPO_URL" "$SRC" || _die "clone failed" fi + # --repo is authoritative on every run, not just the first: retarget the + # cached clone (a stale origin otherwise silently pins every later run + # to the first-ever --repo). + git -C "$SRC" remote set-url origin "$REPO_URL" if git -C "$SRC" fetch origin "$SHA" 2>/dev/null; then # FETCH_HEAD is exactly the requested branch/tag/SHA tip; never # resolve "$SHA" locally here or a branch name would silently hit a @@ -344,6 +363,7 @@ start_container() { fi docker run -d --name "$CONTAINER" \ --privileged --net=host --gpus all --shm-size=8g \ + --ulimit memlock=-1:-1 \ --device /dev/infiniband \ -v "$SRC:/workspaces/cudaqx" \ -v "$WORKDIR/ccache:/root/.ccache" \ @@ -376,6 +396,36 @@ in_ctr() { docker exec "$CONTAINER" bash -lc "$*"; } # --------------------------------------------------------------------------- ROCE_READY=false CH_DEV=""; CH_IP=""; DA_DEV=""; DA_IP="" +NET0=""; NET1="" + +apply_roce_pair_addrs() { + # Idempotent; called before every cpu_roce lane as well as at setup: + # NetworkManager-managed ports drop statically added addresses on each + # DHCP retry cycle (see the README for the permanent nmcli fix). + in_ctr " + set -e + ip link set $NET0 up; ip link set $NET1 up + ip addr replace 10.0.0.1/24 dev $NET0 + ip addr replace 10.0.0.2/24 dev $NET1 + mac0=\$(cat /sys/class/net/$NET0/address) + mac1=\$(cat /sys/class/net/$NET1/address) + ip neigh replace 10.0.0.2 lladdr \$mac1 nud permanent dev $NET0 + ip neigh replace 10.0.0.1 lladdr \$mac0 nud permanent dev $NET1 + " +} + +wait_roce_gids() { + # The IPv4-mapped RoCE GIDs (::ffff:10.0.0.x) appear asynchronously + # after the address add, and the transceivers refuse to start without + # them. Host sysfs and the --net=host container see the same tables. + local i + for i in $(seq 1 20); do + grep -qs 'ffff:0a00:0001' "/sys/class/infiniband/$CH_DEV/ports/1/gids/"* && \ + grep -qs 'ffff:0a00:0002' "/sys/class/infiniband/$DA_DEV/ports/1/gids/"* && return 0 + sleep 0.5 + done + return 1 +} setup_roce_pair() { [[ -z "$ROCE_PAIR" ]] && return 0 @@ -400,21 +450,19 @@ setup_roce_pair() { local dev0="${ROCE_PAIR%%,*}" dev1="${ROCE_PAIR##*,}" [[ -n "$dev0" && -n "$dev1" && "$dev0" != "$dev1" ]] \ || { _err "--roce-pair expects rxe or DEV0,DEV1"; return 1; } - in_ctr " - set -e - net0=\$(ibdev2netdev | awk -v d=$dev0 '\$1==d {print \$5}') - net1=\$(ibdev2netdev | awk -v d=$dev1 '\$1==d {print \$5}') - [ -n \"\$net0\" ] && [ -n \"\$net1\" ] - ip link set \$net0 up; ip link set \$net1 up - ip addr replace 10.0.0.1/24 dev \$net0 - ip addr replace 10.0.0.2/24 dev \$net1 - mac0=\$(cat /sys/class/net/\$net0/address) - mac1=\$(cat /sys/class/net/\$net1/address) - ip neigh replace 10.0.0.2 lladdr \$mac1 nud permanent dev \$net0 - ip neigh replace 10.0.0.1 lladdr \$mac0 nud permanent dev \$net1 - " || { _err "RoCE pair setup failed for $ROCE_PAIR"; return 1; } + # Resolve ibdev -> netdev on the host via sysfs: the image has no + # ibdev2netdev, and --net=host keeps the names identical inside the + # container anyway. + NET0=$(ls "/sys/class/infiniband/$dev0/device/net" 2>/dev/null | head -1) + NET1=$(ls "/sys/class/infiniband/$dev1/device/net" 2>/dev/null | head -1) + [[ -n "$NET0" && -n "$NET1" ]] \ + || { _err "cannot resolve netdevs for $ROCE_PAIR (see /sys/class/infiniband)"; return 1; } CH_DEV="$dev0"; CH_IP=10.0.0.1 DA_DEV="$dev1"; DA_IP=10.0.0.2 + apply_roce_pair_addrs \ + || { _err "RoCE pair setup failed for $ROCE_PAIR"; return 1; } + wait_roce_gids \ + || { _err "IPv4 RoCE GIDs did not appear on $dev0/$dev1"; return 1; } fi ROCE_READY=true _info "cpu_roce pair ready: channel=$CH_DEV/$CH_IP daemon=$DA_DEV/$DA_IP" @@ -429,6 +477,13 @@ teardown_roce_pair() { } roce_env() { # docker exec env flags for the cpu_roce topology + # Re-assert the pair right before each lane: on NetworkManager-managed + # ports the addresses vanish on NM's retry timer, which killed lanes + # minutes after a successful setup. + if [[ "$LIST_ONLY" != true && "$ROCE_PAIR" != rxe && "$ROCE_READY" == true ]]; then + apply_roce_pair_addrs >/dev/null 2>&1 && wait_roce_gids \ + || _info "WARNING: cpu_roce pair re-assert failed; lane may fail" + fi LANE_ENV+=( -e "CUDAQ_CPU_ROCE_TEST_CHANNEL_DEVICE=$CH_DEV" -e "CUDAQ_CPU_ROCE_TEST_CHANNEL_IP=$CH_IP" -e "CUDAQ_CPU_ROCE_TEST_DAEMON_DEVICE=$DA_DEV" @@ -462,6 +517,11 @@ CUDEVICE_HOST="$ARTIFACTS_DIR/cudevice/libcudaq-qec-realtime-cudevice-proprietar NV_QLDPC_PLUGIN_CTR=/artifacts/decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so CUDEVICE_CTR=/artifacts/cudevice/libcudaq-qec-realtime-cudevice-proprietary.a ISING_BUNDLE=/tmp/ising-bundle +# Optional pre-built bundle for tokenless machines (see --hf-token-file help): +# copied into the container so the trt lanes run; only the HF download/export +# path loses coverage, and the ising-prepare SKIP reason says so. +ISING_STAGED_HOST="$ARTIFACTS_DIR/ising-bundle" +ISING_STAGED_CTR=/artifacts/ising-bundle CQ_SRC=/tmp/cudaq-realtime-src have_hf_token() { @@ -485,10 +545,12 @@ ising_ready() { # Count first and convert "no match" into a named SKIP. ctest_cmd() { # regex reason-when-unregistered local regex="$1" reason="$2" + # --timeout only applies to tests without their own TIMEOUT property; it + # bounds a wedged test at 15 min instead of ctest's 1500 s default. echo "cd /workspaces/cudaqx/build && \ n=\$(ctest -N -R '$regex' 2>/dev/null | sed -n 's/^Total Tests: //p'); \ if [ \"\${n:-0}\" -eq 0 ]; then echo 'SKIP: $reason'; exit 77; fi; \ -ctest --output-on-failure -R '$regex'" +ctest --output-on-failure --timeout 900 -R '$regex'" } fpga_dev_flag() { [[ -n "$FPGA_DEVICE" ]] && echo "--device $FPGA_DEVICE"; } @@ -496,7 +558,13 @@ fpga_dev_flag() { [[ -n "$FPGA_DEVICE" ]] && echo "--device $FPGA_DEVICE"; } run_examples_tier() { local d - # -- Ising bundle: fresh gated-HF download + export, every run ---------- + # -- Ising bundle: fresh gated-HF download + export when a token exists; + # otherwise fall back to a bundle staged in the artifacts dir. The copy + # runs outside the lane gate so `--only .../trt_decoder` also benefits. + if ! have_hf_token && [[ "$LIST_ONLY" != true && -f "$ISING_STAGED_HOST/metadata.txt" ]]; then + in_ctr "rm -rf $ISING_BUNDLE && cp -r $ISING_STAGED_CTR $ISING_BUNDLE" \ + || _info "WARNING: staged Ising bundle copy failed" + fi if have_hf_token; then [[ "$LIST_ONLY" == true ]] || LANE_ENV=( -e "HF_TOKEN=$(hf_token)" ) run_lane "examples/ising-prepare" " @@ -506,8 +574,10 @@ run_examples_tier() { [ -n \"\$app\" ] || { echo 'surface_code-4-yaml generator not in the build tree'; exit 1; } python3 $DEMO/prepare_ising_artifacts.py prepare \ --app \"\$app\" --artifacts-dir $ISING_BUNDLE --yes" + elif ising_ready; then + skip_lane "examples/ising-prepare" "no HF token; trt lanes use the staged bundle from $ISING_STAGED_HOST (HF download/export path NOT exercised)" else - skip_lane "examples/ising-prepare" "no HF token (--hf-token-file or HF_TOKEN)" + skip_lane "examples/ising-prepare" "no HF token (--hf-token-file/--hf-token-prompt/HF_TOKEN) and no staged bundle at $ISING_STAGED_HOST" fi # -- qpu-kernel over udp: the no-hardware baseline ----------------------- diff --git a/libs/qec/unittests/realtime/app_examples/CMakeLists.txt b/libs/qec/unittests/realtime/app_examples/CMakeLists.txt index 612323331..3f6713043 100644 --- a/libs/qec/unittests/realtime/app_examples/CMakeLists.txt +++ b/libs/qec/unittests/realtime/app_examples/CMakeLists.txt @@ -365,6 +365,14 @@ if(TARGET cudaq-qec-realtime-decoding-server-cqr) 2 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) + + # The script exits 77 on transports it cannot exercise (per-decoder ring + # endpoints are udp-only), and TIMEOUT bounds a wedged server well below + # the 1500 s ctest default. + set_tests_properties( + app_examples.surface_code-1-cqr-two-process-test-distance-3 + app_examples.surface_code-1-cqr-two-process-test-distance-3-num-logical-2 + PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 300) endif() endif() diff --git a/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh b/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh index c09301342..f1c883a37 100644 --- a/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh +++ b/libs/qec/unittests/realtime/app_examples/surface_code-1-cqr-two-process-test.sh @@ -53,6 +53,16 @@ NUM_ROUNDS=$6 DECODER_TYPE=${7:-multi_error_lut} NUM_LOGICAL=${8:-1} +# Device-scoped endpoint args exist only for udp (udp-port.=): over +# cpu_roce the app refuses --num_logical > 1, and the server would then wedge +# in ring 1's blocking rendezvous accept(). Skip with a named reason +# (SKIP_RETURN_CODE 77 in the ctest registration). +if [[ "${QEC_DECODING_SERVER_TRANSPORT:-udp}" != "udp" && "$NUM_LOGICAL" -gt 1 ]]; then + echo "SKIP: per-decoder ring endpoints exercised over udp only (no" \ + "device-scoped ${QEC_DECODING_SERVER_TRANSPORT} channel args)" + exit 77 +fi + export CUDAQ_DEFAULT_SIMULATOR=stim NUM_SHOTS=1000 @@ -86,10 +96,23 @@ fi QEC_DECODING_SERVER_STATS=1 $SERVER_PATH "${SERVER_ARGS[@]}" \ > $SERVER_LOG 2>&1 & SERVER_PID=$! -cleanup() { +# TERM first; escalate to KILL after a 5 s grace. The cpu_roce bridge's +# rendezvous connect() blocks in accept() where the server's shutdown flag +# is never checked, and a wedged server must fail the test fast rather +# than sit out the whole ctest timeout. +stop_server() { kill -TERM $SERVER_PID 2>/dev/null || true + for _ in $(seq 1 50); do + kill -0 $SERVER_PID 2>/dev/null || break + sleep 0.1 + done + if kill -0 $SERVER_PID 2>/dev/null; then + echo "WARNING: decoding_server ignored SIGTERM; sending SIGKILL" + kill -KILL $SERVER_PID 2>/dev/null || true + fi wait $SERVER_PID 2>/dev/null || true } +cleanup() { stop_server; } trap cleanup EXIT # Wait for the READY line and parse the port. @@ -133,8 +156,7 @@ QEC_DECODING_SERVER_PORT=$SERVER_PORT \ |& tee $APP_LOG # [4] Stop the server and collect its dispatch count. -kill -TERM $SERVER_PID -wait $SERVER_PID 2>/dev/null || true +stop_server trap - EXIT num_non_zero_values=$(grep "Number of non-zero values measured :" $APP_LOG | awk -F': ' '{print $2}') diff --git a/libs/qec/unittests/realtime/test_decoding_server.cpp b/libs/qec/unittests/realtime/test_decoding_server.cpp index 2d21d1797..34ae9c684 100644 --- a/libs/qec/unittests/realtime/test_decoding_server.cpp +++ b/libs/qec/unittests/realtime/test_decoding_server.cpp @@ -489,6 +489,14 @@ __qpu__ std::int64_t dual_decoding_server_kernel() { } TEST(DecodingServerTwoProcess, TwoProcessHostDispatchDualDecoders) { + // Shared-wire demux is udp-only: a multi-decoder server opens one ring + // per decoder, and on rendezvous transports (cpu_roce) its post-READY + // connect() blocks until every ring is dialed -- this caller dials one + // endpoint (see decoding_server.cpp [3]), so the test would hang, not + // fail. Multi-ring coverage is TwoProcessPerDecoderRings (also udp-only + // today: device-scoped cpu_roce endpoint args do not exist yet). + if (env_or("QEC_DECODING_SERVER_TRANSPORT", "udp") != "udp") + GTEST_SKIP() << "shared-wire dual decoders demux over udp only"; // Two identical 3-bit-identity pymatching decoders (ids 0 and 1) in one // server -- one per logical qubit. const std::string config_path = From dff10a48da24a49b954443324326062b56598d24 Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Tue, 11 Aug 2026 14:17:58 -0500 Subject: [PATCH 3/9] [QEC] hw CI: drop the HSB BlueFlame tweak, fix the Spark FPGA port and bridge-lane pacing Three follow-ups from continued Spark validation: - Drop the holoscan-sensor-bridge DOCA_SEND_BLUE_FLAME=0 build tweak. The gpu_roce UAR failures that motivated it were environmental, not a platform property: pointed at the FPGA-cabled port, the stock BlueFlame build passes every FPGA lane end-to-end. - README: the Spark FPGA invocation must name the port actually cabled to the FPGA (roceP2p1s0f0 on the current lab Spark). A wrong-but-linked --fpga-device still passes the HSB control plane and BRAM verification but fails with "ILA: captured 0 of N expected samples". - Pass --spacing 100 to the gpu-roce-qldpc-bridge lane: at the playback tool's default 10 us inter-shot spacing the ILA verification deterministically undercounts (194/500 on the Spark, reproducible across doorbell modes, ports, and reboots); at 100 us -- the same pacing the demo lanes use -- it captures 500/500 with all 100 shots verified and corrections matched. - Clarify the runner's build-phase messages: "Container build complete" read as if the docker image had just finished building; the messages now distinguish the layer-cached image build from the per-commit source build (cudaq-realtime + cudaqx + demo) that runs inside the container. With these, the FPGA configuration certifies clean on the Spark: every lane PASSes or SKIPs with a named reason, zero failures. Signed-off-by: Chuck Ketcham --- HANDOFF-gb200-hw-ci.md | 141 ++++++++++++++++++++++ docker/decoding-server/dev.Dockerfile | 6 - docker/decoding-server/hw_ci/README.md | 6 +- docker/decoding-server/hw_ci/run_hw_ci.sh | 12 +- 4 files changed, 153 insertions(+), 12 deletions(-) create mode 100644 HANDOFF-gb200-hw-ci.md diff --git a/HANDOFF-gb200-hw-ci.md b/HANDOFF-gb200-hw-ci.md new file mode 100644 index 000000000..69733f7ba --- /dev/null +++ b/HANDOFF-gb200-hw-ci.md @@ -0,0 +1,141 @@ +# Handoff: decoding-server hardware CI — GB200 validation + +**TEMPORARY FILE — do not commit. DELETE THIS FILE once everything is +passing on the GB200** (Chuck's instruction: it exists only for this +bring-up). + +You are a fresh Claude Code instance on the GB200 HOST. The hardware CI +(`docker/decoding-server/hw_ci/run_hw_ci.sh`) has been fully validated on a +DGX Spark in both of its cable configurations; your job is the GB200 leg: +verify prerequisites, run the suite in the GB200 configuration, and +report/fix what breaks. + +## Ground rules (the user is Chuck; these are standing preferences) + +- **Never run `git commit` or `git push`** (or `git add`/`git reset` that + alters staging). Put code changes directly into the working tree of the + clone you are told to use and present a proposed commit message in your + reply; Chuck makes every commit himself. +- Work from a clone of **Chuck's fork** (`cketcham2333/cudaqx`), branch + `decoding-server-hw-ci`. +- Cheap probe before long runs: `--list`, then a single udp/pymatching + lane, then one FPGA lane, then the full set. +- Answer Chuck's clarifying questions standalone; don't bundle them with + new question prompts. +- When Chuck rules something out about his own environment ("I never + updated X"), take it as ground truth and redirect the investigation. +- Do not generalize hardware capability from one device in one boot state: + sweep all devices and re-test across reboots before concluding "this + platform can't do X". (This lesson was paid for on the Spark.) + +## State of the work + +Branch `decoding-server-hw-ci` on the fork. Expected tip: `c7c740d` ("Fix +decoding-server hw CI issues found in DGX Spark host validation") plus a +follow-up commit (BlueFlame sed removal, Spark FPGA port doc, bridge-lane +`--spacing 100`, build-message clarity). **Verify before starting** that +your checkout has the follow-up: the gpu-roce-qldpc-bridge lane in +`run_hw_ci.sh` must pass `--spacing 100`, and `dev.Dockerfile` must NOT +sed `DOCA_SEND_BLUE_FLAME`. If those are missing, ask Chuck whether the +follow-up commit was pushed. + +## Already validated on the Spark (do not redo) + +- Loopback config (`--no-fpga --roce-pair rocep1s0f0,rocep1s0f1`): + 13 passed / 0 failed / 9 named skips. +- FPGA config (`--fpga-device roceP2p1s0f0`): 15 passed / 0 failed / + 7 named skips — all four decoders over udp and the FPGA source, + device_graph GPU dispatch, hsb-fpga-server on both wires, the + gpu-roce-qldpc-bridge, ising-prepare (gated HF download + on-GPU + export), and the qldpc-graph / mixed-dispatch ctests. +- Tokenless trt fallback (staged `ising-bundle/` in the artifacts dir) + and `--hf-token-prompt` were added and validated for public-account use. + +## Your job on the GB200 (cheapest first) + +1. Prereqs: docker + nvidia-container-toolkit, user in `docker` group; + `ghcr.io/nvidia/cudaqx-dev` pulls anonymously (verify with + `docker manifest inspect` of the pin-matched tag: shortref from + `jq -r .cudaq.ref .cudaq_version | head -c8`, tag + `-arm64-cu13.0`); ~60 GB free disk. +2. `sudo modprobe rdma_rxe` for the SoftRoCE mode (see Known gaps below + before spending time here). +3. Proprietary artifacts staged (default `/opt/nvqlink-lab-artifacts`): + `decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so` and + `cudevice/libcudaq-qec-realtime-cudevice-proprietary.a` — must cover + sm_100 (GB200). Do NOT reuse the Spark's copies blindly (those came + from a GB10 dev build; verify arch coverage or rebuild). +4. **Ask Chuck which IB device faces the FPGA** — do not guess from link + state. On the Spark, a wrong-but-linked `--fpga-device` passed the HSB + control plane and BRAM verification but failed with + `ILA: captured 0 of N expected samples`. That symptom = wrong port. +5. Probe: `run_hw_ci.sh --list`, then + `--only 'examples/qpu-kernel/udp/pymatching'`, then one FPGA lane + (`--only 'examples/fpga/cpu_roce/pymatching'`), then the full run: + + ``` + ./docker/decoding-server/hw_ci/run_hw_ci.sh \ + --repo --sha decoding-server-hw-ci \ + --roce-pair rxe --fpga-device \ + --artifacts-dir /opt/nvqlink-lab-artifacts + ``` + + HF token: the GB200 runs under a PUBLIC account. Preferred: stage the + pre-built Ising bundle at `/ising-bundle/` (copy from + the Spark: `~/nvqlink-lab-artifacts/ising-bundle/`) and run tokenless — + the trt lanes then run and ising-prepare SKIPs with a named reason. + If Chuck wants the HF download path exercised, use `--hf-token-prompt` + (interactive, nothing written to disk); never store a token file on a + shared account. + +## Known gaps / expected results on GB200 + +- `--roce-pair rxe` SKIPs the two-process cpu_roce lanes: the image's + Mellanox ibverbs-providers has no rxe userspace provider (README + documents it; the skip is loud). Fixing it means building the rxe + provider from Mellanox rdma-core 2601 source into the image — a known + follow-up, not a bug you introduced. +- `extra/ctest/ai-decoder-fp8` reports "not registered" everywhere until + `CUDAQX_QEC_ENABLE_REALTIME_PIPELINE` is re-enabled upstream. +- Multi-decoder two-process tests (DualDecoders, num-logical-2, + PerDecoderRings) skip by name off udp — device-scoped cpu_roce endpoint + args don't exist yet upstream. +- 64 KiB-page kernels (GB200 `-64k`): `derive_page_sizes` defaults the + slot size to 512 and the device_graph lane rounds up to a + host-page-compatible value. This logic has NEVER run on a real 64K + host — watch the device-graph lane's geometry first. + +## Hard-won lessons that may recur (from the Spark bring-up) + +- **BlueFlame UAR failures are transient/stateful, not a platform + property.** Symptom: gpu_roce lanes die at start with + `gpu_roce_transceiver.cpp:314 ... Failed to create UAR: DOCA Driver + call failure`. On the Spark this hit one ConnectX card for a whole + afternoon and cleared on reboot, while NONCACHE always worked. Do NOT + patch HSB (a NONCACHE sed was tried and deliberately removed). If it + appears: sweep ALL devices with a minimal `doca_uar_create` reproducer + (open device, try BLUEFLAME then NONCACHE), try another boot, and + report. Upstream ask on record: runtime BF->NONCACHE fallback in HSB. +- NetworkManager (if the GB200 runs it) silently drops statically + assigned pair addresses on its DHCP retry timer; the runner re-asserts + addresses + waits for IPv4 RoCE GIDs before each cpu_roce lane, so this + should be handled — the permanent fix is + `sudo nmcli device set managed no`. +- Root-owned clones: git needs `safe.directory` entries for BOTH the + clone path and `/.git` (local-path `--repo` accesses the repo by + its `.git` path). +- `decoding_server` ignores SIGTERM while blocked in a cpu_roce + rendezvous accept(); the test scripts carry TERM->KILL escalation and + ctest lanes run with `--timeout 900`, so a wedge costs minutes, not + hours. If you see a wedged server, that upstream bug is already on + record. +- The runner's `ibdev2netdev` comes from `mlnx-ofed-kernel-utils` (modern + `mlnx-tools` no longer ships it); the image installs it. + +## Reporting + +Summarize the PASS/FAIL/SKIP table per configuration, plus any fixes made +(working-tree diffs + a proposed commit message; never commit). Compare +against the Spark tables above: the GB200's expected deltas are the two +rxe skips (instead of cpu_roce passes) and FPGA lanes running on GB200's +port. When everything passes and Chuck confirms, **delete this file**. diff --git a/docker/decoding-server/dev.Dockerfile b/docker/decoding-server/dev.Dockerfile index 43c4f3ceb..5de6ea58c 100644 --- a/docker/decoding-server/dev.Dockerfile +++ b/docker/decoding-server/dev.Dockerfile @@ -159,12 +159,6 @@ RUN set -e; \ /add_subdirectory(udp_transmitter)/d; /add_subdirectory(emulator)/d; \ /add_subdirectory(sig_gen)/d; /add_subdirectory(sig_viewer)/d' \ src/hololink/operators/CMakeLists.txt; \ - # In-tree mlx5 drivers (e.g. the DGX Spark's -nvidia kernel, no OFED) - # reject BlueFlame UAR allocation -- doca_uar_create fails and every - # gpu_roce lane dies at transceiver start. NONCACHE doorbells are - # functionally equivalent for the CI lanes. - sed -i 's/#define DOCA_SEND_BLUE_FLAME 1/#define DOCA_SEND_BLUE_FLAME 0/' \ - src/hololink/operators/gpu_roce_transceiver/gpu_roce_transceiver_common.hpp; \ export CUDA_NATIVE_ARCH=${cuda_native_arch}; \ cmake -G Ninja -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md index e02d0e396..bb03bc2bb 100644 --- a/docker/decoding-server/hw_ci/README.md +++ b/docker/decoding-server/hw_ci/README.md @@ -93,8 +93,10 @@ as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. # DGX Spark, single cable in loopback mode (port0 <-> port1, no FPGA): ./run_hw_ci.sh --sha --no-fpga --roce-pair rocep1s0f0,rocep1s0f1 -# DGX Spark, single cable in FPGA mode: -./run_hw_ci.sh --sha --fpga-device rocep1s0f0 +# DGX Spark, single cable in FPGA mode (use the port actually cabled to the +# FPGA -- on the current lab Spark that is roceP2p1s0f0; a wrong-but-linked +# port fails with "ILA: captured 0 of N expected samples"): +./run_hw_ci.sh --sha --fpga-device roceP2p1s0f0 ``` `--repo` also accepts a local clone path (avoids pushing while iterating). diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh index 1d146ae5f..50e6c03fc 100755 --- a/docker/decoding-server/hw_ci/run_hw_ci.sh +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -699,12 +699,15 @@ run_extra_tier() { elif [[ ( ! -f "$NV_QLDPC_PLUGIN_HOST" || ! -f "$CUDEVICE_HOST" ) && "$LIST_ONLY" != true ]]; then skip_lane "extra/gpu-roce-qldpc-bridge" "missing nv-qldpc plugin and/or cudevice archive" else + # --spacing 100: at the playback tool's default 10 us inter-shot + # spacing the ILA verification deterministically undercounts + # (194/500 on the Spark); 100 us matches the demo lanes' pacing. run_lane "extra/gpu-roce-qldpc-bridge" " set -e bridge=\$(find /workspaces/cudaqx/build -name gpu_roce_qldpc_graph_decoder_bridge -type f | head -1) [ -n \"\$bridge\" ] || { echo 'SKIP: bridge executable not built'; exit 77; } bash $gbridge --setup-network $(fpga_dev_flag) --bridge-ip $BRIDGE_IP --fpga-ip $FPGA_IP \ - --page-size $PAGE_SIZE_DG \ + --page-size $PAGE_SIZE_DG --spacing 100 \ --cuda-qx-dir /workspaces/cudaqx --cuda-quantum-dir $CQ_SRC \ --hsb-dir /opt/holoscan-sensor-bridge \ --proprietary-archive $CUDEVICE_CTR --nv-qldpc-plugin $NV_QLDPC_PLUGIN_CTR" @@ -755,14 +758,15 @@ check_cudaq_pin start_container setup_roce_pair || _info "continuing without a cpu_roce pair" -_info "Building cudaq-realtime + cudaqx + demo binaries in the container" +_info "Image ready; compiling the commit under test inside the container:" +_info " cudaq-realtime + cudaqx + demo binaries (log: $LOG_DIR/container_build.log)" build_log="$LOG_DIR/container_build.log" if ! in_ctr "bash /workspaces/cudaqx/docker/decoding-server/hw_ci/container_build.sh \ --cuda-arch $CUDA_ARCH" >"$build_log" 2>&1; then tail -40 "$build_log" >&2 - _die "container build failed; full log: $build_log" + _die "in-container source build failed; full log: $build_log" fi -_info "Container build complete" +_info "Source build complete (cudaq-realtime + cudaqx + demo)" run_all_lanes print_summary From e454dfe9b848f332eb11d6b98ffd9096425c50ba Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Tue, 11 Aug 2026 20:43:24 +0000 Subject: [PATCH 4/9] [QEC] hw CI: SoftRoCE on DOCA-OFED hosts (GB200): rxe provider in-image, OFED-compat rdma_rxe fallback On hosts whose ib_core comes from DOCA/MLNX-OFED DKMS (GB200 #2, 6.17.0-1008-nvidia-64k + DOCA-OFED 25.10), --roce-pair rxe had two blockers: - the distro rdma_rxe.ko cannot load against the OFED ib_core ("disagrees about version of symbol ib_*"), and OFED dropped the rxe driver from its source tree. Stage a patched upstream v6.17 rxe driver in the image (hw_ci/rxe-ofed/: three compat deltas + an iser-style external-module makefile); in rxe mode the runner mounts /lib/modules + /usr/src read-only, builds it against the host's ofa_kernel tree inside the privileged container, and insmods it -- only when the distro modprobe fails. - Mellanox ibverbs-providers ships only the mlx5 provider, and its provider ABI (rdmav59) differs from Ubuntu's rdma-core. Build librxe from the same Mellanox rdma-core source release (DOCA SOURCES bundle, sha256-pinned; the rxe provider is shipped there but if(0)-disabled in CMakeLists.txt) into the image. Validated on GB200: module loads against DOCA-OFED 25.10; provider opens rxe devices in-container. Signed-off-by: Chuck Ketcham --- docker/decoding-server/dev.Dockerfile | 66 +++++++++++++-- docker/decoding-server/hw_ci/README.md | 16 ++-- docker/decoding-server/hw_ci/run_hw_ci.sh | 26 +++++- .../decoding-server/hw_ci/rxe-ofed/README.md | 64 +++++++++++++++ .../decoding-server/hw_ci/rxe-ofed/makefile | 72 +++++++++++++++++ .../hw_ci/rxe-ofed/ofed-compat.patch | 81 +++++++++++++++++++ .../hw_ci/rxe-ofed/prepare-src.sh | 32 ++++++++ .../hw_ci/rxe-ofed/rxe_ofed_compat.h | 15 ++++ 8 files changed, 357 insertions(+), 15 deletions(-) create mode 100644 docker/decoding-server/hw_ci/rxe-ofed/README.md create mode 100644 docker/decoding-server/hw_ci/rxe-ofed/makefile create mode 100644 docker/decoding-server/hw_ci/rxe-ofed/ofed-compat.patch create mode 100755 docker/decoding-server/hw_ci/rxe-ofed/prepare-src.sh create mode 100644 docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h diff --git a/docker/decoding-server/dev.Dockerfile b/docker/decoding-server/dev.Dockerfile index 5de6ea58c..5bbe49b26 100644 --- a/docker/decoding-server/dev.Dockerfile +++ b/docker/decoding-server/dev.Dockerfile @@ -52,12 +52,12 @@ ARG cuda_native_arch=100 # Build tools + RDMA userspace. # NOTE: the base image ships Mellanox OFED's rdma-core fork, whose # ibverbs-providers outranks Ubuntu's and contains ONLY the mlx5 provider -- -# no rxe (SoftRoCE). The apt line below therefore keeps the Mellanox -# package, and `--roce-pair rxe` runs will SKIP with a named reason until a -# rxe provider matching the Mellanox libibverbs ABI is built into the image -# (see the hw_ci README). perftest (ib_write_bw) is for fabric smoke tests. -# The `sudo` binary must exist because the example scripts' network helpers -# invoke it literally (a no-op when already root). +# no rxe (SoftRoCE). The apt line below keeps the Mellanox package; the +# missing rxe provider is built from the matching Mellanox source in a later +# layer (see "SoftRoCE support" at the end of this file). perftest +# (ib_write_bw) is for fabric smoke tests. The `sudo` binary must exist +# because the example scripts' network helpers invoke it literally (a no-op +# when already root). # --------------------------------------------------------------------------- RUN apt-get update && apt-get install -y --no-install-recommends \ ninja-build curl pkg-config jq \ @@ -187,5 +187,59 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && test -x /usr/sbin/ibdev2netdev && command -v patchelf +# --------------------------------------------------------------------------- +# SoftRoCE support, part 1: the rxe userspace provider. The Mellanox +# rdma-core fork installed above ships only the mlx5 provider, and its +# provider ABI (rdmav59) differs from Ubuntu's rdma-core, so the Ubuntu +# ibverbs-providers package cannot supply librxe either. Build it from the +# SAME Mellanox rdma-core source release (the DOCA SOURCES bundle; the rxe +# provider is shipped there but `if (0)`-disabled in CMakeLists.txt) and +# install just the provider .so. The dpkg-query / PABI derivations keep +# this layer loudly consistent with whatever rdma-core version the DOCA +# repo actually installed. kmod supplies insmod for part 2. +# --------------------------------------------------------------------------- +ARG mlnx_ofed_src_ver=26.01-1.0.0.0 +ARG mlnx_ofed_src_sha256=ed5597a547c2d5bb858b43f2305ec19f539bc70c4e5ed75aa6c6897a715568d3 +RUN set -e; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + cmake ninja-build pkg-config patch kmod \ + libnl-3-dev libnl-route-3-dev libudev-dev; \ + RDMA_CORE_VER=$(dpkg-query -W -f '${Version}' rdma-core); \ + RDMA_CORE_VER=${RDMA_CORE_VER%-*}; \ + MULTIARCH=$(gcc -print-multiarch); \ + PABI=$(basename /usr/lib/$MULTIARCH/libibverbs/libmlx5-rdmav*.so \ + | sed 's|libmlx5-rdmav\([0-9]*\)\.so|\1|'); \ + tmp=$(mktemp -d); cd "$tmp"; \ + curl -fsSLO "https://linux.mellanox.com/public/repo/doca/3.3.0/SOURCES/mlnx_ofed/MLNX_OFED_SRC-debian-${mlnx_ofed_src_ver}.tgz"; \ + echo "${mlnx_ofed_src_sha256} MLNX_OFED_SRC-debian-${mlnx_ofed_src_ver}.tgz" | sha256sum -c -; \ + tar xzf "MLNX_OFED_SRC-debian-${mlnx_ofed_src_ver}.tgz" \ + "MLNX_OFED_SRC-${mlnx_ofed_src_ver}/SOURCES/rdma-core_${RDMA_CORE_VER}.orig.tar.gz"; \ + tar xzf "MLNX_OFED_SRC-${mlnx_ofed_src_ver}/SOURCES/rdma-core_${RDMA_CORE_VER}.orig.tar.gz"; \ + cd "rdma-core-${RDMA_CORE_VER}"; \ + sed -i 's|^add_subdirectory(providers/mlx5)$|add_subdirectory(providers/mlx5)\nadd_subdirectory(providers/rxe)|' \ + CMakeLists.txt; \ + cmake -GNinja -S . -B build -DNO_MAN_PAGES=1 -DNO_PYVERBS=1 >/dev/null; \ + ninja -C build "librxe-rdmav${PABI}.so"; \ + install -m 644 "build/lib/librxe-rdmav${PABI}.so" "/usr/lib/$MULTIARCH/libibverbs/"; \ + printf 'driver rxe\n' > /etc/libibverbs.d/rxe.driver; \ + cd /; rm -rf "$tmp"; \ + apt-get clean && rm -rf /var/lib/apt/lists/*; \ + test -f "/usr/lib/$MULTIARCH/libibverbs/librxe-rdmav${PABI}.so" + +# --------------------------------------------------------------------------- +# SoftRoCE support, part 2: OFED-compat rdma_rxe kernel module SOURCE (see +# hw_ci/rxe-ofed/README.md). On hosts whose ib_core comes from DOCA/MLNX- +# OFED DKMS (e.g. GB200 #2), the distro's in-tree rdma_rxe.ko cannot load +# (symbol CRC mismatch), so run_hw_ci.sh's rxe mode builds this patched +# copy of the upstream rxe driver against the host's kernel headers + +# ofa_kernel tree (mounted at /lib/modules and /usr/src) inside the +# privileged container and insmods it. The image only STAGES the patched +# source; the compile is per-host at container setup time. +# --------------------------------------------------------------------------- +ARG rxe_kernel_ref=v6.17 +COPY hw_ci/rxe-ofed /opt/rxe-ofed +RUN /opt/rxe-ofed/prepare-src.sh "${rxe_kernel_ref}" /opt/rxe-ofed/src + ENV HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=/opt/holoscan-sensor-bridge \ HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=/opt/holoscan-sensor-bridge/build diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md index bb03bc2bb..ae7db0646 100644 --- a/docker/decoding-server/hw_ci/README.md +++ b/docker/decoding-server/hw_ci/README.md @@ -42,12 +42,16 @@ as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. `docker login ghcr.io` (to pull `ghcr.io/nvidia/cudaqx-dev`). 2. For SoftRoCE mode (`--roce-pair rxe`): `sudo modprobe rdma_rxe`, persisted via `echo rdma_rxe | sudo tee /etc/modules-load.d/rdma_rxe.conf`. - **Known gap:** the dev image inherits Mellanox OFED's `ibverbs-providers`, - which ships only the mlx5 userspace provider — no rxe. Until a rxe - provider matching that libibverbs ABI is built into the image, rxe setup - fails its `ibv_devinfo` preflight and the two-process cpu_roce lanes SKIP - with a named reason. Machines with a real loopback-cabled port pair - (`--roce-pair DEV0,DEV1`) are unaffected. + On hosts whose `ib_core` comes from DOCA/MLNX-OFED DKMS (e.g. a GB200 + with `doca-ofed` installed), the distro module refuses to load + (`disagrees about version of symbol ib_*`); the runner then builds an + OFED-compat `rdma_rxe` from the source staged in the image and loads it + from the privileged container automatically — no host setup needed, but + the module does not persist across host reboots (see + `rxe-ofed/README.md`). The image also builds the rxe userspace provider + that Mellanox's `ibverbs-providers` omits, from the matching Mellanox + rdma-core source release, so `ibv_devinfo` can open rxe devices + in-container. 3. FPGA cabled/flashed and reachable (defaults: NIC 192.168.0.1/24, FPGA 192.168.0.2). Machines whose single cable is wired as a loopback pair instead run with `--no-fpga --roce-pair DEV0,DEV1`. diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh index 50e6c03fc..c9be9c511 100755 --- a/docker/decoding-server/hw_ci/run_hw_ci.sh +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -361,6 +361,13 @@ start_container() { else _info "Artifacts dir $ARTIFACTS_DIR absent; proprietary lanes will SKIP" fi + # rxe mode may need to build the OFED-compat rdma_rxe module in-container + # (see setup_roce_pair), which compiles against the HOST kernel headers + # and ofa_kernel tree. + local rxe_mount=() + if [[ "$ROCE_PAIR" == "rxe" ]]; then + rxe_mount=(-v /lib/modules:/lib/modules:ro -v /usr/src:/usr/src:ro) + fi docker run -d --name "$CONTAINER" \ --privileged --net=host --gpus all --shm-size=8g \ --ulimit memlock=-1:-1 \ @@ -368,6 +375,7 @@ start_container() { -v "$SRC:/workspaces/cudaqx" \ -v "$WORKDIR/ccache:/root/.ccache" \ ${artifacts_mount[@]+"${artifacts_mount[@]}"} \ + ${rxe_mount[@]+"${rxe_mount[@]}"} \ "$IMAGE" sleep infinity >/dev/null || _die "docker run failed" } @@ -431,10 +439,22 @@ setup_roce_pair() { [[ -z "$ROCE_PAIR" ]] && return 0 if [[ "$ROCE_PAIR" == "rxe" ]]; then if ! grep -qw rdma_rxe /proc/modules; then - if ! sudo -n modprobe rdma_rxe 2>/dev/null; then - _err "rdma_rxe is not loaded; run 'sudo modprobe rdma_rxe' and retry" + sudo -n modprobe rdma_rxe 2>/dev/null || true + fi + if ! grep -qw rdma_rxe /proc/modules; then + # DOCA/MLNX-OFED DKMS hosts: the distro rdma_rxe cannot bind to + # the OFED ib_core (symbol CRC mismatch), so build the staged + # OFED-compat copy against the host headers mounted at + # /lib/modules + /usr/src and load it from the privileged + # container (see rxe-ofed/README.md). + _info "distro rdma_rxe not loadable; building the OFED-compat module in-container" + in_ctr "make -C /opt/rxe-ofed/src >/tmp/rxe-ofed-build.log 2>&1 \ + && insmod /opt/rxe-ofed/src/rdma_rxe.ko" || { + _err "OFED-compat rdma_rxe build/load failed; last lines of the build log:" + in_ctr "tail -15 /tmp/rxe-ofed-build.log" >&2 || true + _err "(full log: docker exec $CONTAINER cat /tmp/rxe-ofed-build.log)" return 1 - fi + } fi in_ctr " ip link add hwci-dummy0 type dummy 2>/dev/null || true diff --git a/docker/decoding-server/hw_ci/rxe-ofed/README.md b/docker/decoding-server/hw_ci/rxe-ofed/README.md new file mode 100644 index 000000000..782288cd2 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/README.md @@ -0,0 +1,64 @@ +# SoftRoCE (rxe) on MLNX/DOCA-OFED hosts + +Two independent gaps stop `--roce-pair rxe` from working out of the box on +hosts that run DOCA-OFED (validated on a GB200, `6.17.0-1008-nvidia-64k` + +DOCA-OFED 25.10): + +1. **Kernel:** the OFED DKMS stack replaces `ib_core` (in + `/lib/modules/*/updates/dkms/`), whose exported symbol CRCs differ from + the in-tree build, so the distro's `rdma_rxe.ko` fails to load with + `disagrees about version of symbol ib_*` (err -22) — and MLNX OFED + dropped the rxe driver from its own source tree (only the uapi header + remains in `ofa_kernel`). +2. **Userspace:** the Mellanox rdma-core fork's `ibverbs-providers` ships + only the mlx5 provider. Its provider ABI (`rdmav59`) differs from + Ubuntu's rdma-core, so the Ubuntu package can't supply `librxe` either; + in their source release the rxe provider exists but is `if (0)`-disabled + in `CMakeLists.txt`. + +## What this directory provides (gap 1) + +- `ofed-compat.patch` — three small deltas that make the **upstream v6.17** + `drivers/infiniband/sw/rxe` compile against the OFED compat API + (GPL-2.0, like the sources it patches): + - drop the `struct ib_dmah *` parameter from `rxe_reg_user_mr` + (upstream v6.16+ API; OFED's `ib_device_ops` predates it), + - `umem_odp->map.pfn_list` → `umem_odp->pfn_list` (OFED keeps the + pre-`hmm_dma_map` layout), + - (via `rxe_ofed_compat.h`, injected by the makefile with `-include`) + the two ODP capability bits OFED's enum lacks. +- `makefile` — external-module build against + `/usr/src/ofa_kernel[-dkms]//` (include chain + + `KBUILD_EXTRA_SYMBOLS`, the same pattern MLNX's own iser/isert DKMS + packages use). Refuses to build without an OFED tree: on inbox-rdma + hosts the distro `modprobe rdma_rxe` is the right module. +- `prepare-src.sh` — sparse-clones the pinned upstream tag (default + `v6.17`, image build-arg `rxe_kernel_ref`), applies the patch + (`--fuzz=0`: drift fails the image build), and drops the makefile + + compat header next to the sources. + +`dev.Dockerfile` stages the patched source at `/opt/rxe-ofed/src` at image +build. The `.ko` itself is **per-host** and is built at container setup +time by `run_hw_ci.sh` (rxe mode mounts `/usr/src` + `/lib/modules` +read-only, builds in-container, and `insmod`s from the privileged +container). Nothing is loaded on hosts where the distro module works — +that is always tried first. + +Gap 2 is fixed directly in `dev.Dockerfile`: it builds `librxe-rdmav` +from the **same Mellanox rdma-core source release** as the installed +packages (the `MLNX_OFED_SRC-debian` bundle from the DOCA `SOURCES` repo, +sha256-pinned) and installs it into the image's libibverbs provider +directory. + +## Caveats + +- The loaded module does not survive a host reboot; the runner re-builds + and re-loads on demand. For a permanent host install, DKMS-ify this + directory (out of scope here). +- The pinned kernel ref should roughly match the host kernel generation; + the OFED tree it compiles against, however, is whatever the host has + installed. A very different host kernel will surface as loud compile + errors at setup time (the lanes then SKIP, never silently). +- If a future DOCA release restores rxe (kernel or userspace), the distro + `modprobe` / packaged provider win automatically and this machinery goes + dormant. diff --git a/docker/decoding-server/hw_ci/rxe-ofed/makefile b/docker/decoding-server/hw_ci/rxe-ofed/makefile new file mode 100644 index 000000000..c7220ac28 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/makefile @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Build the (patched) upstream rxe SoftRoCE driver as an external module +# against the MLNX/DOCA-OFED ofa_kernel compat headers + Module.symvers, so +# it binds to the OFED DKMS ib_core instead of the in-tree one. +# +# Why: on hosts with doca-ofed / mlnx-ofed-kernel-dkms installed (e.g. +# GB200), ib_core & friends come from /lib/modules/*/updates/dkms and export +# different symbol CRCs than the in-tree build, so the distro's rdma_rxe.ko +# fails to load with "disagrees about version of symbol ib_*" -- and OFED +# dropped the rxe driver from its own source tree. The external-module +# pattern below (OFED include chain + KBUILD_EXTRA_SYMBOLS) is copied from +# the makefiles of MLNX OFED's own out-of-tree DKMS packages (iser/isert). +# +# This makefile, rxe_ofed_compat.h and ofed-compat.patch are expected to sit +# next to (and the patch already applied to) the upstream +# drivers/infiniband/sw/rxe sources -- prepare-src.sh stages exactly that. +# GNU make prefers this lowercase 'makefile'; kbuild's sub-make reads the +# upstream capital 'Makefile' for the object list. + +KVER ?= $(shell uname -r) +ARCH_M := $(shell uname -m) +OFA_DIR ?= $(shell test -d /usr/src/ofa_kernel-dkms/$(ARCH_M) && echo /usr/src/ofa_kernel-dkms/$(ARCH_M) || echo /usr/src/ofa_kernel/$(ARCH_M)) +OFA := $(OFA_DIR)/$(KVER) +K_BUILD ?= /lib/modules/$(KVER)/build + +ifneq ($(wildcard $(OFA)/Module.symvers),) +include $(OFA)/configure.mk.kernel +endif + +autoconf_h := $(shell /bin/ls -1 $(K_BUILD)/include/*/autoconf.h 2>/dev/null | head -1) +kconfig_h := $(shell /bin/ls -1 $(K_BUILD)/include/*/kconfig.h 2>/dev/null | head -1) +ifneq ($(kconfig_h),) +KCONFIG_H = -include $(kconfig_h) +endif + +LINUXINCLUDE=\ + -include $(autoconf_h) \ + $(KCONFIG_H) \ + -include $(OFA)/include/linux/compat-2.6.h \ + -include $(CURDIR)/rxe_ofed_compat.h \ + -I$(OFA)/include \ + -I$(OFA)/include/uapi \ + $(BACKPORT_INCLUDES) \ + -I$$(srctree)/arch/$$(SRCARCH)/include \ + -I$$(objtree)/arch/$$(SRCARCH)/include/generated \ + -I$$(objtree)/include \ + -I$$(srctree)/arch/$$(SRCARCH)/include/uapi \ + -I$$(objtree)/arch/$$(SRCARCH)/include/generated/uapi \ + -I$$(srctree)/include \ + -I$$(srctree)/include/uapi \ + -I$$(objtree)/include/generated/uapi \ + # + +default: +ifneq ($(wildcard $(OFA)/Module.symvers),) + $(MAKE) -C $(K_BUILD) O=$(K_BUILD) M=$(CURDIR) \ + KBUILD_EXTRA_SYMBOLS=$(OFA)/Module.symvers \ + CONFIG_RDMA_RXE=m \ + LINUXINCLUDE='$(LINUXINCLUDE)' \ + modules +else + @echo "error: $(OFA_DIR)/$(KVER) has no Module.symvers." >&2; \ + echo "This OFED-compat build is only for MLNX/DOCA-OFED DKMS hosts;" >&2; \ + echo "on inbox-rdma hosts load the distro module: sudo modprobe rdma_rxe" >&2; \ + echo "(if run from the hw_ci container: are /usr/src and /lib/modules" >&2; \ + echo "mounted from the host?)" >&2; \ + exit 1 +endif + +clean: + rm -f *.o *.ko *.mod *.mod.c modules.order Module.symvers .*.cmd diff --git a/docker/decoding-server/hw_ci/rxe-ofed/ofed-compat.patch b/docker/decoding-server/hw_ci/rxe-ofed/ofed-compat.patch new file mode 100644 index 000000000..fe68ed0e5 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/ofed-compat.patch @@ -0,0 +1,81 @@ +--- a/rxe_verbs.c 2026-08-11 20:26:52.811000000 +0000 ++++ b/rxe_verbs.c 2026-08-11 20:16:55.914000000 +0000 +@@ -1269,9 +1269,11 @@ + return ERR_PTR(err); + } + ++/* OFED-compat: DOCA-OFED 25.10's ib_device_ops.reg_user_mr has no ++ * struct ib_dmah parameter (added upstream in v6.16+); drop it and the ++ * corresponding reject-if-set check from the v6.17 source. */ + static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start, + u64 length, u64 iova, int access, +- struct ib_dmah *dmah, + struct ib_udata *udata) + { + struct rxe_dev *rxe = to_rdev(ibpd->device); +@@ -1279,9 +1281,6 @@ + struct rxe_mr *mr; + int err, cleanup_err; + +- if (dmah) +- return ERR_PTR(-EOPNOTSUPP); +- + if (access & ~RXE_ACCESS_SUPPORTED_MR) { + rxe_err_pd(pd, "access = %#x not supported (%#x)\n", access, + RXE_ACCESS_SUPPORTED_MR); +--- a/rxe_odp.c 2026-08-11 20:26:52.908892000 +0000 ++++ b/rxe_odp.c 2026-08-11 20:18:05.338000000 +0000 +@@ -27,7 +27,7 @@ + start = max_t(u64, ib_umem_start(umem_odp), range->start); + end = min_t(u64, ib_umem_end(umem_odp), range->end); + +- /* update umem_odp->map.pfn_list */ ++ /* update umem_odp->pfn_list */ + ib_umem_odp_unmap_dma_pages(umem_odp, start, end); + + mutex_unlock(&umem_odp->umem_mutex); +@@ -137,7 +137,7 @@ + while (addr < iova + length) { + idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; + +- if (!(umem_odp->map.pfn_list[idx] & HMM_PFN_VALID)) { ++ if (!(umem_odp->pfn_list[idx] & HMM_PFN_VALID)) { + need_fault = true; + break; + } +@@ -201,7 +201,7 @@ + while (length > 0) { + u8 *src, *dest; + +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[idx]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[idx]); + user_va = kmap_local_page(page); + + src = (dir == RXE_TO_MR_OBJ) ? addr : user_va; +@@ -288,7 +288,7 @@ + } + + idx = rxe_odp_iova_to_index(umem_odp, iova); +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[idx]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[idx]); + + va = kmap_local_page(page); + +@@ -347,7 +347,7 @@ + index = rxe_odp_iova_to_index(umem_odp, iova); + page_offset = rxe_odp_iova_to_page_offset(umem_odp, iova); + +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[index]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[index]); + + bytes = min_t(unsigned int, length, + mr_page_size(mr) - page_offset); +@@ -396,7 +396,7 @@ + } + + index = rxe_odp_iova_to_index(umem_odp, iova); +- page = hmm_pfn_to_page(umem_odp->map.pfn_list[index]); ++ page = hmm_pfn_to_page(umem_odp->pfn_list[index]); + + va = kmap_local_page(page); + /* Do atomic write after all prior operations have completed */ diff --git a/docker/decoding-server/hw_ci/rxe-ofed/prepare-src.sh b/docker/decoding-server/hw_ci/rxe-ofed/prepare-src.sh new file mode 100755 index 000000000..330da2d59 --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/prepare-src.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Stage the pinned upstream rxe (SoftRoCE) driver source with the +# OFED-compat patch applied, ready for a per-host `make` at container setup +# time (see the sibling makefile and README.md for why). +# +# Runs at dev-image build time (dev.Dockerfile); can also be run manually. +# +# prepare-src.sh [kernel-ref] [dest-dir] +set -euo pipefail + +ref=${1:-v6.17} +dest=${2:-/opt/rxe-ofed/src} +here=$(cd "$(dirname "$0")" && pwd) + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +# Sparse partial clone: only the rxe directory's blobs are fetched. +git clone --depth 1 --branch "$ref" --filter=tree:0 --sparse --quiet \ + https://github.com/torvalds/linux.git "$tmp/linux" +git -C "$tmp/linux" sparse-checkout set drivers/infiniband/sw/rxe + +mkdir -p "$dest" +cp "$tmp/linux/drivers/infiniband/sw/rxe/"* "$dest/" +test -f "$dest/rxe.c" # sparse checkout sanity + +# --fuzz=0: any drift between the pinned ref and the patch must fail the +# image build loudly rather than half-apply. +patch -d "$dest" -p1 --fuzz=0 < "$here/ofed-compat.patch" + +cp "$here/rxe_ofed_compat.h" "$here/makefile" "$dest/" +echo "rxe-ofed: staged $ref + ofed-compat.patch at $dest" diff --git a/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h b/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h new file mode 100644 index 000000000..a6636bf8b --- /dev/null +++ b/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * The MLNX/DOCA-OFED compat enum ib_odp_transport_cap_bits ends at + * IB_ODP_SUPPORT_SRQ_RECV (1 << 5); these two bits exist upstream in v6.17 + * (values mirror uapi IB_UVERBS_ODP_SUPPORT_* exactly). Injected via + * -include from the accompanying makefile so the upstream rxe sources need + * no edit for this. + */ +#ifndef RXE_OFED_COMPAT_H +#define RXE_OFED_COMPAT_H + +#define IB_ODP_SUPPORT_FLUSH (1 << 6) +#define IB_ODP_SUPPORT_ATOMIC_WRITE (1 << 7) + +#endif From dd9ae052f5d52ec5ce91b572b1cc9e0abbc15b08 Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Tue, 11 Aug 2026 20:50:57 +0000 Subject: [PATCH 5/9] [QEC] hw CI: fix two SoftRoCE integration fallouts found on GB200 - dev.Dockerfile: do not apt-install cmake for the rxe provider build. noble's cmake 3.28 shadows the base image's cmake 4.x at /bin/cmake, and the in-container source build (cudaq-realtime) requires >= 4.0; rdma-core needs only >= 3.18.1, which the base cmake satisfies. - run_hw_ci.sh: bind-mount /dev/infiniband instead of --device. --device snapshots char devices at container creation, so the uverbs node of the rxe device that setup_roce_pair creates AFTER container start never appeared inside, and ibv_devinfo reported "IB device 'hwci_rxe0' wasn't found". A bind mount is live; the privileged container loses no device-cgroup allowance. Signed-off-by: Chuck Ketcham --- docker/decoding-server/dev.Dockerfile | 6 +++++- docker/decoding-server/hw_ci/run_hw_ci.sh | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/decoding-server/dev.Dockerfile b/docker/decoding-server/dev.Dockerfile index 5bbe49b26..47e5bfa19 100644 --- a/docker/decoding-server/dev.Dockerfile +++ b/docker/decoding-server/dev.Dockerfile @@ -200,10 +200,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # --------------------------------------------------------------------------- ARG mlnx_ofed_src_ver=26.01-1.0.0.0 ARG mlnx_ofed_src_sha256=ed5597a547c2d5bb858b43f2305ec19f539bc70c4e5ed75aa6c6897a715568d3 +# NOTE: no apt cmake here -- the base image ships cmake 4.x under +# /usr/local/cmake-*/bin, and noble's cmake 3.28 would shadow it at +# /bin/cmake, breaking the later in-container source build (cudaq-realtime +# requires cmake >= 4.0). rdma-core needs >= 3.18.1, satisfied by the base. RUN set -e; \ apt-get update; \ apt-get install -y --no-install-recommends \ - cmake ninja-build pkg-config patch kmod \ + ninja-build pkg-config patch kmod \ libnl-3-dev libnl-route-3-dev libudev-dev; \ RDMA_CORE_VER=$(dpkg-query -W -f '${Version}' rdma-core); \ RDMA_CORE_VER=${RDMA_CORE_VER%-*}; \ diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh index c9be9c511..b4d580ab3 100755 --- a/docker/decoding-server/hw_ci/run_hw_ci.sh +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -368,10 +368,15 @@ start_container() { if [[ "$ROCE_PAIR" == "rxe" ]]; then rxe_mount=(-v /lib/modules:/lib/modules:ro -v /usr/src:/usr/src:ro) fi + # /dev/infiniband is a live bind mount, NOT --device: --device snapshots + # the char devices at container creation, so the uverbs node of an rxe + # device created later by setup_roce_pair would never appear in the + # container and libibverbs would not find the device. The container is + # privileged, so no device-cgroup allowance is lost by the switch. docker run -d --name "$CONTAINER" \ --privileged --net=host --gpus all --shm-size=8g \ --ulimit memlock=-1:-1 \ - --device /dev/infiniband \ + -v /dev/infiniband:/dev/infiniband \ -v "$SRC:/workspaces/cudaqx" \ -v "$WORKDIR/ccache:/root/.ccache" \ ${artifacts_mount[@]+"${artifacts_mount[@]}"} \ From ba9b36cd1e90fa9eca9c004ac58a77f466848864 Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Tue, 11 Aug 2026 22:00:03 +0000 Subject: [PATCH 6/9] [QEC] hw CI: GB200 validation complete -- lab-friendly defaults, drop dead lane and bring-up note GB200 validation finished at dd9ae05: 21 passed / 0 failed across udp, cpu_roce-over-SoftRoCE, and FPGA (mlx5_4), with ai-decoder-fp8 the only (permanent) skip. Follow-ups from that bring-up: - --roce-pair defaults to rxe (SoftRoCE self-loop); the old omitted-means-skip behavior moves to an explicit --roce-pair none - --fpga-device defaults to mlx5_4 (the GB200 lab wiring) - drop --repo: the commit under test is always cloned from the repo this script lives in, which also makes local unpushed commits directly testable - drop the ai-decoder-fp8 lane: the ctest is never registered until CUDAQX_QEC_ENABLE_REALTIME_PIPELINE returns upstream, so the permanent named skip carries no signal; re-add the lane when the pipeline lands - delete the GB200 bring-up handoff note, per its own instructions now that the machine is validated Signed-off-by: Chuck Ketcham --- HANDOFF-gb200-hw-ci.md | 141 ---------------------- docker/decoding-server/hw_ci/README.md | 13 +- docker/decoding-server/hw_ci/run_hw_ci.sh | 28 +++-- 3 files changed, 22 insertions(+), 160 deletions(-) delete mode 100644 HANDOFF-gb200-hw-ci.md diff --git a/HANDOFF-gb200-hw-ci.md b/HANDOFF-gb200-hw-ci.md deleted file mode 100644 index 69733f7ba..000000000 --- a/HANDOFF-gb200-hw-ci.md +++ /dev/null @@ -1,141 +0,0 @@ -# Handoff: decoding-server hardware CI — GB200 validation - -**TEMPORARY FILE — do not commit. DELETE THIS FILE once everything is -passing on the GB200** (Chuck's instruction: it exists only for this -bring-up). - -You are a fresh Claude Code instance on the GB200 HOST. The hardware CI -(`docker/decoding-server/hw_ci/run_hw_ci.sh`) has been fully validated on a -DGX Spark in both of its cable configurations; your job is the GB200 leg: -verify prerequisites, run the suite in the GB200 configuration, and -report/fix what breaks. - -## Ground rules (the user is Chuck; these are standing preferences) - -- **Never run `git commit` or `git push`** (or `git add`/`git reset` that - alters staging). Put code changes directly into the working tree of the - clone you are told to use and present a proposed commit message in your - reply; Chuck makes every commit himself. -- Work from a clone of **Chuck's fork** (`cketcham2333/cudaqx`), branch - `decoding-server-hw-ci`. -- Cheap probe before long runs: `--list`, then a single udp/pymatching - lane, then one FPGA lane, then the full set. -- Answer Chuck's clarifying questions standalone; don't bundle them with - new question prompts. -- When Chuck rules something out about his own environment ("I never - updated X"), take it as ground truth and redirect the investigation. -- Do not generalize hardware capability from one device in one boot state: - sweep all devices and re-test across reboots before concluding "this - platform can't do X". (This lesson was paid for on the Spark.) - -## State of the work - -Branch `decoding-server-hw-ci` on the fork. Expected tip: `c7c740d` ("Fix -decoding-server hw CI issues found in DGX Spark host validation") plus a -follow-up commit (BlueFlame sed removal, Spark FPGA port doc, bridge-lane -`--spacing 100`, build-message clarity). **Verify before starting** that -your checkout has the follow-up: the gpu-roce-qldpc-bridge lane in -`run_hw_ci.sh` must pass `--spacing 100`, and `dev.Dockerfile` must NOT -sed `DOCA_SEND_BLUE_FLAME`. If those are missing, ask Chuck whether the -follow-up commit was pushed. - -## Already validated on the Spark (do not redo) - -- Loopback config (`--no-fpga --roce-pair rocep1s0f0,rocep1s0f1`): - 13 passed / 0 failed / 9 named skips. -- FPGA config (`--fpga-device roceP2p1s0f0`): 15 passed / 0 failed / - 7 named skips — all four decoders over udp and the FPGA source, - device_graph GPU dispatch, hsb-fpga-server on both wires, the - gpu-roce-qldpc-bridge, ising-prepare (gated HF download + on-GPU - export), and the qldpc-graph / mixed-dispatch ctests. -- Tokenless trt fallback (staged `ising-bundle/` in the artifacts dir) - and `--hf-token-prompt` were added and validated for public-account use. - -## Your job on the GB200 (cheapest first) - -1. Prereqs: docker + nvidia-container-toolkit, user in `docker` group; - `ghcr.io/nvidia/cudaqx-dev` pulls anonymously (verify with - `docker manifest inspect` of the pin-matched tag: shortref from - `jq -r .cudaq.ref .cudaq_version | head -c8`, tag - `-arm64-cu13.0`); ~60 GB free disk. -2. `sudo modprobe rdma_rxe` for the SoftRoCE mode (see Known gaps below - before spending time here). -3. Proprietary artifacts staged (default `/opt/nvqlink-lab-artifacts`): - `decoder-plugins/libcudaq-qec-nv-qldpc-decoder.so` and - `cudevice/libcudaq-qec-realtime-cudevice-proprietary.a` — must cover - sm_100 (GB200). Do NOT reuse the Spark's copies blindly (those came - from a GB10 dev build; verify arch coverage or rebuild). -4. **Ask Chuck which IB device faces the FPGA** — do not guess from link - state. On the Spark, a wrong-but-linked `--fpga-device` passed the HSB - control plane and BRAM verification but failed with - `ILA: captured 0 of N expected samples`. That symptom = wrong port. -5. Probe: `run_hw_ci.sh --list`, then - `--only 'examples/qpu-kernel/udp/pymatching'`, then one FPGA lane - (`--only 'examples/fpga/cpu_roce/pymatching'`), then the full run: - - ``` - ./docker/decoding-server/hw_ci/run_hw_ci.sh \ - --repo --sha decoding-server-hw-ci \ - --roce-pair rxe --fpga-device \ - --artifacts-dir /opt/nvqlink-lab-artifacts - ``` - - HF token: the GB200 runs under a PUBLIC account. Preferred: stage the - pre-built Ising bundle at `/ising-bundle/` (copy from - the Spark: `~/nvqlink-lab-artifacts/ising-bundle/`) and run tokenless — - the trt lanes then run and ising-prepare SKIPs with a named reason. - If Chuck wants the HF download path exercised, use `--hf-token-prompt` - (interactive, nothing written to disk); never store a token file on a - shared account. - -## Known gaps / expected results on GB200 - -- `--roce-pair rxe` SKIPs the two-process cpu_roce lanes: the image's - Mellanox ibverbs-providers has no rxe userspace provider (README - documents it; the skip is loud). Fixing it means building the rxe - provider from Mellanox rdma-core 2601 source into the image — a known - follow-up, not a bug you introduced. -- `extra/ctest/ai-decoder-fp8` reports "not registered" everywhere until - `CUDAQX_QEC_ENABLE_REALTIME_PIPELINE` is re-enabled upstream. -- Multi-decoder two-process tests (DualDecoders, num-logical-2, - PerDecoderRings) skip by name off udp — device-scoped cpu_roce endpoint - args don't exist yet upstream. -- 64 KiB-page kernels (GB200 `-64k`): `derive_page_sizes` defaults the - slot size to 512 and the device_graph lane rounds up to a - host-page-compatible value. This logic has NEVER run on a real 64K - host — watch the device-graph lane's geometry first. - -## Hard-won lessons that may recur (from the Spark bring-up) - -- **BlueFlame UAR failures are transient/stateful, not a platform - property.** Symptom: gpu_roce lanes die at start with - `gpu_roce_transceiver.cpp:314 ... Failed to create UAR: DOCA Driver - call failure`. On the Spark this hit one ConnectX card for a whole - afternoon and cleared on reboot, while NONCACHE always worked. Do NOT - patch HSB (a NONCACHE sed was tried and deliberately removed). If it - appears: sweep ALL devices with a minimal `doca_uar_create` reproducer - (open device, try BLUEFLAME then NONCACHE), try another boot, and - report. Upstream ask on record: runtime BF->NONCACHE fallback in HSB. -- NetworkManager (if the GB200 runs it) silently drops statically - assigned pair addresses on its DHCP retry timer; the runner re-asserts - addresses + waits for IPv4 RoCE GIDs before each cpu_roce lane, so this - should be handled — the permanent fix is - `sudo nmcli device set managed no`. -- Root-owned clones: git needs `safe.directory` entries for BOTH the - clone path and `/.git` (local-path `--repo` accesses the repo by - its `.git` path). -- `decoding_server` ignores SIGTERM while blocked in a cpu_roce - rendezvous accept(); the test scripts carry TERM->KILL escalation and - ctest lanes run with `--timeout 900`, so a wedge costs minutes, not - hours. If you see a wedged server, that upstream bug is already on - record. -- The runner's `ibdev2netdev` comes from `mlnx-ofed-kernel-utils` (modern - `mlnx-tools` no longer ships it); the image installs it. - -## Reporting - -Summarize the PASS/FAIL/SKIP table per configuration, plus any fixes made -(working-tree diffs + a proposed commit message; never commit). Compare -against the Spark tables above: the GB200's expected deltas are the two -rxe skips (instead of cpu_roce passes) and FPGA lanes running on GB200's -port. When everything passes and Chuck confirms, **delete this file**. diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md index ae7db0646..513dbbeaf 100644 --- a/docker/decoding-server/hw_ci/README.md +++ b/docker/decoding-server/hw_ci/README.md @@ -12,10 +12,14 @@ CUDA-graph path is unreachable there). as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. ``` -./run_hw_ci.sh --sha --roce-pair rxe --hf-token-file ~/.hf_token +./run_hw_ci.sh --sha --hf-token-prompt ./run_hw_ci.sh --list # show the lane set ``` +The commit is cloned from the repo the script itself lives in, so local +(unpushed) commits are testable directly. SoftRoCE (`--roce-pair rxe`) and +the GB200 lab FPGA port (`--fpga-device mlx5_4`) are the defaults. + ## Lanes * **examples tier** — the shipped `examples/qec/realtime_decoding_demo` @@ -90,9 +94,8 @@ as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. ## Per-machine invocations ```bash -# GB200 #1 (FPGA on one port, no free port pair -> SoftRoCE for two-process): -./run_hw_ci.sh --sha --roce-pair rxe --fpga-device rocep1s0f0 \ - --hf-token-file ~/.hf_token +# GB200 (lab default wiring: FPGA on mlx5_4, SoftRoCE for two-process): +./run_hw_ci.sh --sha --hf-token-prompt # DGX Spark, single cable in loopback mode (port0 <-> port1, no FPGA): ./run_hw_ci.sh --sha --no-fpga --roce-pair rocep1s0f0,rocep1s0f1 @@ -102,8 +105,6 @@ as `PASS`, `FAIL`, or `SKIP(reason)`; `--strict` turns skips into failures. # port fails with "ILA: captured 0 of N expected samples"): ./run_hw_ci.sh --sha --fpga-device roceP2p1s0f0 ``` - -`--repo` also accepts a local clone path (avoids pushing while iterating). If that clone is owned by another user — e.g. it was created from a devcontainer running as root — git refuses to serve it ("dubious ownership"); allow it once with diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh index b4d580ab3..1adfb7f06 100755 --- a/docker/decoding-server/hw_ci/run_hw_ci.sh +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -36,7 +36,10 @@ SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Defaults # --------------------------------------------------------------------------- SHA="" -REPO_URL="https://github.com/NVIDIA/cudaqx.git" +# The repo under test is the one this script lives in; a fresh clone of it +# is made at the requested --sha. +REPO_URL="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel 2>/dev/null)" +[[ -n "$REPO_URL" ]] || { echo "error: cannot resolve the containing git repo (is the script inside a clone?)" >&2; exit 1; } WORKDIR="$HOME/.cache/cudaqx-hw-ci" ARTIFACTS_DIR="/opt/nvqlink-lab-artifacts" TIER="all" # examples | extra | all @@ -50,10 +53,10 @@ BASE_IMAGE="" # resolved from the pin unless given BUILD_BASE=false CUDA_VERSION="13.0" CUDA_ARCH="" # auto-detected unless given (Spark=121, GB200=100) -ROCE_PAIR="" # rxe | DEV0,DEV1 ; empty = skip cpu_roce pair lanes +ROCE_PAIR="rxe" # rxe | DEV0,DEV1 | (empty via --roce-pair none = skip cpu_roce pair lanes) HF_TOKEN_FILE="" HF_TOKEN_PROMPT=false -FPGA_DEVICE="" # ConnectX IB device facing the FPGA +FPGA_DEVICE="mlx5_4" # ConnectX IB device facing the FPGA (GB200 lab wiring) BRIDGE_IP="192.168.0.1" FPGA_IP="192.168.0.2" PAGE_SIZE="" # default derived from the host page size @@ -65,7 +68,8 @@ print_usage() { Usage: run_hw_ci.sh --sha SHA [options] --sha SHA cudaqx commit (or branch/tag) to clone, build, test - --repo URL cudaqx clone URL (default: $REPO_URL) + (cloned from the repo containing this script: + $REPO_URL) --workdir DIR clone/build/log root (default: $WORKDIR) --artifacts-dir DIR proprietary artifacts, bind-mounted RO at /artifacts (default: $ARTIFACTS_DIR); layout: @@ -94,11 +98,12 @@ Hardware: --cuda-arch N CUDA architecture (default: auto via nvidia-smi; DGX Spark GB10=121, GB200=100) --roce-pair rxe SoftRoCE self-loop for the two-process cpu_roce - lanes (GB200 #1: no free ConnectX port pair), OR - --roce-pair DEV0,DEV1 a real loopback-cabled IB device pair (DGX Spark); - omitted => those lanes SKIP - --fpga-device DEV ConnectX IB device facing the FPGA (default: the - test scripts auto-detect / rocep1s0f0) + lanes (the default; no free ConnectX port pair + needed), OR + --roce-pair DEV0,DEV1 a real loopback-cabled IB device pair (DGX Spark), + OR --roce-pair none => those lanes SKIP + --fpga-device DEV ConnectX IB device facing the FPGA + (default: $FPGA_DEVICE, the GB200 lab wiring) --bridge-ip IP server-side NIC IP (default $BRIDGE_IP) --fpga-ip IP FPGA IP (default $FPGA_IP) --page-size N RDMA ring slot size (default: 384, or 512 on @@ -121,7 +126,6 @@ EOF while [[ $# -gt 0 ]]; do case "$1" in --sha) SHA="$2"; shift ;; - --repo) REPO_URL="$2"; shift ;; --workdir) WORKDIR="$2"; shift ;; --artifacts-dir) ARTIFACTS_DIR="$2"; shift ;; --tier) TIER="$2"; shift ;; @@ -135,7 +139,7 @@ while [[ $# -gt 0 ]]; do --build-base) BUILD_BASE=true ;; --cuda-version) CUDA_VERSION="$2"; shift ;; --cuda-arch) CUDA_ARCH="$2"; shift ;; - --roce-pair) ROCE_PAIR="$2"; shift ;; + --roce-pair) ROCE_PAIR="$2"; [[ "$ROCE_PAIR" == none ]] && ROCE_PAIR=""; shift ;; --fpga-device) FPGA_DEVICE="$2"; shift ;; --bridge-ip) BRIDGE_IP="$2"; shift ;; --fpga-ip) FPGA_IP="$2"; shift ;; @@ -714,8 +718,6 @@ run_extra_tier() { "$(ctest_cmd '^test_realtime_qldpc_graph_decoding$' 'not registered (needs nv-qldpc plugin + cudevice archive at configure)')" run_lane "extra/ctest/mixed-dispatch" \ "$(ctest_cmd 'app_examples.surface_code-4-yaml-mixed-dispatch' 'not registered (needs cudevice archive at configure)')" - run_lane "extra/ctest/ai-decoder-fp8" \ - "$(ctest_cmd 'test_ai_decoder_quantized_onnx' 'not registered')" # -- gpu_roce bridge cross-check (same data plane, no server layers) ------ local gbridge=/workspaces/cudaqx/libs/qec/unittests/utils/gpu_roce_qldpc_graph_decoder_test.sh From c5029d76405ad4bb91b0410ee26904d216c72b6f Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Tue, 11 Aug 2026 22:18:04 +0000 Subject: [PATCH 7/9] add nvqlink-lab-artifacts parameter to the README Signed-off-by: Chuck Ketcham --- docker/decoding-server/hw_ci/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md index 513dbbeaf..49e047918 100644 --- a/docker/decoding-server/hw_ci/README.md +++ b/docker/decoding-server/hw_ci/README.md @@ -94,8 +94,10 @@ the GB200 lab FPGA port (`--fpga-device mlx5_4`) are the defaults. ## Per-machine invocations ```bash -# GB200 (lab default wiring: FPGA on mlx5_4, SoftRoCE for two-process): -./run_hw_ci.sh --sha --hf-token-prompt +# GB200 (lab default wiring: FPGA on mlx5_4, SoftRoCE for two-process; +# artifacts staged in the home directory rather than the /opt default): +./run_hw_ci.sh --sha --hf-token-prompt \ + --artifacts-dir ~/nvqlink-lab-artifacts # DGX Spark, single cable in loopback mode (port0 <-> port1, no FPGA): ./run_hw_ci.sh --sha --no-fpga --roce-pair rocep1s0f0,rocep1s0f1 From 55bb81712f2d19cb5f2ec12ba46e07075b62dd7a Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Tue, 11 Aug 2026 22:25:30 +0000 Subject: [PATCH 8/9] clang format Signed-off-by: Chuck Ketcham --- docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h b/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h index a6636bf8b..9c1741eab 100644 --- a/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h +++ b/docker/decoding-server/hw_ci/rxe-ofed/rxe_ofed_compat.h @@ -9,7 +9,7 @@ #ifndef RXE_OFED_COMPAT_H #define RXE_OFED_COMPAT_H -#define IB_ODP_SUPPORT_FLUSH (1 << 6) -#define IB_ODP_SUPPORT_ATOMIC_WRITE (1 << 7) +#define IB_ODP_SUPPORT_FLUSH (1 << 6) +#define IB_ODP_SUPPORT_ATOMIC_WRITE (1 << 7) #endif From fcbfc248333cd70797af24824a85fc8b4b84bc24 Mon Sep 17 00:00:00 2001 From: Chuck Ketcham Date: Thu, 13 Aug 2026 21:52:08 +0000 Subject: [PATCH 9/9] [QEC] hw CI: serialize runs per machine; label the trt lanes with their model Two users running the CI on one machine collided: the container name was derived from the commit SHA alone (docker rm -f would kill the other's run), and the FPGA, SoftRoCE objects (hwci-dummy0/hwci_rxe0), and RoCE pair addressing are host-global regardless of SHA. - take a host-wide flock on /tmp/cudaqx-hw-ci.lock for every real run (--list is exempt); a second invocation fails fast naming the active run's user/pid/sha, or queues behind it with --lock-wait - the kernel releases the flock when the holder dies, so crashes can never leave a stale lock; for a live-but-wedged holder (including an orphaned child that inherited the fd) --force-unlock kills whatever fuser says is actually holding the lock, with a sudo hint when the holder belongs to another user - container name gains $USER (hwci-$USER-$SHORT_SHA) so a --keep-container debugging container survives someone else later testing the same SHA - rename the trt lanes to trt_decoder(ising): the plugin is the generic TensorRT decoder, but the engine under test is the exported Ising model, and the summary table should say so - README: one-run-at-a-time + recovery documented; lane list clarified Signed-off-by: Chuck Ketcham --- docker/decoding-server/hw_ci/README.md | 15 ++++- docker/decoding-server/hw_ci/run_hw_ci.sh | 76 +++++++++++++++++++++-- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/docker/decoding-server/hw_ci/README.md b/docker/decoding-server/hw_ci/README.md index 49e047918..530b9a5ae 100644 --- a/docker/decoding-server/hw_ci/README.md +++ b/docker/decoding-server/hw_ci/README.md @@ -24,7 +24,8 @@ the GB200 lab FPGA port (`--fpga-device mlx5_4`) are the defaults. * **examples tier** — the shipped `examples/qec/realtime_decoding_demo` driver: all 4 decoders (pymatching, multi_error_lut, nv-qldpc-decoder, - trt_decoder) over `udp` (baseline), `cpu_roce` two-process (RoCE pair), + and the TensorRT plugin running the exported Ising model — lane name + `trt_decoder(ising)`) over `udp` (baseline), `cpu_roce` two-process (RoCE pair), and the FPGA source (`cpu_roce` host dispatch ×4, `device_graph` nv-qldpc ×1); plus `ising-prepare`, which downloads the gated Hugging Face Ising model and rebuilds the TRT artifact bundle **on every run** — a FAIL @@ -84,7 +85,17 @@ the GB200 lab FPGA port (`--fpga-device mlx5_4`) are the defaults. download/export path was not exercised. No token and no staged bundle => the ising/trt lanes SKIP. 6. ~60 GB free disk for image layers and build trees. -7. NetworkManager-managed ports (the DGX OS default) silently drop the +7. **One run per machine at a time.** The FPGA (SIF/BRAM/ILA), the + SoftRoCE objects, and the RoCE pair addressing are host-global, so the + runner serializes itself with a host-wide lock + (`/tmp/cudaqx-hw-ci.lock`). A second invocation fails immediately, + naming the active run's user/pid/sha; `--lock-wait` queues behind it + instead. A crashed or killed run releases the lock automatically (the + kernel drops it with the process — the lock file itself is inert), so + only a live-but-wedged run can hold others out: `--force-unlock` kills + whatever actually holds the lock and proceeds (for another user's + wedged run: `sudo fuser -vk /tmp/cudaqx-hw-ci.lock`). +8. NetworkManager-managed ports (the DGX OS default) silently drop the runner's statically assigned `10.0.0.x` pair addresses on NM's DHCP retry timer. The runner re-asserts the addresses and waits for the IPv4-mapped RoCE GIDs before every cpu_roce lane, which is normally diff --git a/docker/decoding-server/hw_ci/run_hw_ci.sh b/docker/decoding-server/hw_ci/run_hw_ci.sh index 1adfb7f06..efae15453 100755 --- a/docker/decoding-server/hw_ci/run_hw_ci.sh +++ b/docker/decoding-server/hw_ci/run_hw_ci.sh @@ -62,6 +62,8 @@ FPGA_IP="192.168.0.2" PAGE_SIZE="" # default derived from the host page size KEEP_CONTAINER=false NO_FPGA=false +LOCK_WAIT=false # queue behind an active run instead of failing +FORCE_UNLOCK=false # kill a wedged lock holder, then run print_usage() { cat <&2; print_usage >&2; exit 1 ;; esac @@ -171,6 +181,56 @@ if [[ "$HF_TOKEN_PROMPT" == true && "$LIST_ONLY" != true ]]; then export HF_TOKEN fi +# --------------------------------------------------------------------------- +# Host-wide run lock: one hw CI run per machine at a time. The FPGA (SIF +# registers/BRAM/ILA), the SoftRoCE objects (hwci-dummy0/hwci_rxe0), and the +# RoCE pair addressing are all host-global, so concurrent runs would corrupt +# each other. flock(2) is released by the kernel the instant the holding +# process dies (crash, Ctrl-C, kill -9), so no failure can leave a stale +# lock; the lock FILE persisting on disk is inert. The only lockout is a +# live-but-wedged holder -- --force-unlock kills whatever actually holds the +# lock (fuser sees the real holders, including orphaned children that +# inherited the fd), and the lock-busy message shows them. +# --------------------------------------------------------------------------- +LOCK_FILE="${HWCI_LOCK_FILE:-/tmp/cudaqx-hw-ci.lock}" +LOCK_FD="" + +acquire_run_lock() { + # 0666 regardless of umask: every user must be able to lock it. + ( umask 000; : >>"$LOCK_FILE" ) 2>/dev/null + exec {LOCK_FD}<>"$LOCK_FILE" || _die "cannot open lock file $LOCK_FILE" + + if ! flock -n "$LOCK_FD"; then + if [[ "$FORCE_UNLOCK" == true ]]; then + _info "--force-unlock: killing the current holder(s) of $LOCK_FILE:" + fuser -v "$LOCK_FILE" || true + fuser -k -TERM "$LOCK_FILE" >/dev/null 2>&1 || true + local i + for i in 1 2 3 4 5; do sleep 1; flock -n "$LOCK_FD" && break; done + if ! flock -n "$LOCK_FD"; then + fuser -k -KILL "$LOCK_FILE" >/dev/null 2>&1 || true + sleep 1 + fi + flock -n "$LOCK_FD" || _die "--force-unlock failed -- the holder likely belongs to another user; escalate with: sudo fuser -vk $LOCK_FILE" + elif [[ "$LOCK_WAIT" == true ]]; then + _info "another hw CI run is active ($(tr -d '\n' <"$LOCK_FILE" 2>/dev/null)); waiting for it to finish ..." + flock "$LOCK_FD" || _die "waiting for the lock failed" + else + local holder pid orphan="" + holder=$(tr -d '\n' <"$LOCK_FILE" 2>/dev/null) + pid=$(sed -n 's/.*pid=\([0-9]\+\).*/\1/p' <<<"$holder") + [[ -n "$pid" && ! -d "/proc/$pid" ]] && \ + orphan=" -- its recorded pid is gone, so an orphaned child still holds the lock (fuser -v $LOCK_FILE shows it)" + _err "another hw CI run is active on this machine: ${holder:-}$orphan" + _die "rerun with --lock-wait to queue behind it, or --force-unlock to kill a wedged holder" + fi + fi + # Advisory holder info for the messages above; the flock itself is the gate. + printf 'user=%s pid=%s sha=%s since=%s\n' "$USER" $$ "$SHA" "$(date -Is)" >"$LOCK_FILE" 2>/dev/null || true +} + +[[ "$LIST_ONLY" == true ]] || acquire_run_lock + # --------------------------------------------------------------------------- # Lane bookkeeping. Every lane lands in the summary exactly once as # PASS / FAIL / SKIP(reason). Lanes run strictly sequentially: everything @@ -356,7 +416,10 @@ Wait for build_dev.yaml to publish the new pin's image, pass --base-image, or us } start_container() { - CONTAINER="hwci-$SHORT_SHA" + # $USER in the name so a --keep-container debugging container from one + # user is not docker-rm'd by another user later testing the same SHA + # (concurrent runs are already excluded by the host-wide lock). + CONTAINER="hwci-$USER-$SHORT_SHA" docker rm -f "$CONTAINER" >/dev/null 2>&1 || true mkdir -p "$WORKDIR/ccache" local artifacts_mount=() @@ -524,6 +587,11 @@ roce_env() { # docker exec env flags for the cpu_roce topology # the device_graph ring (64 slots) must total a multiple of the host page # size, so its value rounds up to the compatible one. # --------------------------------------------------------------------------- +# Lane label: the trt_decoder plugin runs the exported Ising model in these +# lanes -- make that visible in lane names and the summary table. The +# --decoder argument stays 'trt_decoder' (the plugin name). +lane_label() { [[ "$1" == trt_decoder ]] && echo 'trt_decoder(ising)' || echo "$1"; } + derive_page_sizes() { local host_page; host_page=$(getconf PAGESIZE) if [[ -z "$PAGE_SIZE" ]]; then @@ -611,7 +679,7 @@ run_examples_tier() { # -- qpu-kernel over udp: the no-hardware baseline ----------------------- for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do - local name="examples/qpu-kernel/udp/$d" extra="" + local name="examples/qpu-kernel/udp/$(lane_label "$d")" extra="" case "$d" in nv-qldpc-decoder) [[ -f "$NV_QLDPC_PLUGIN_HOST" || "$LIST_ONLY" == true ]] \ @@ -628,7 +696,7 @@ run_examples_tier() { # No --setup-network: the runner configured the pair itself (the demo's # helper resolves ports via ibdev2netdev, which cannot see rxe devices). for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do - local name="examples/qpu-kernel/cpu_roce/$d" extra="" + local name="examples/qpu-kernel/cpu_roce/$(lane_label "$d")" extra="" [[ "$ROCE_READY" == true || "$LIST_ONLY" == true ]] \ || { skip_lane "$name" "no cpu_roce pair (--roce-pair not set / setup failed)"; continue; } case "$d" in @@ -646,7 +714,7 @@ run_examples_tier() { # -- FPGA source, cpu_roce wire, host dispatch --------------------------- for d in pymatching multi_error_lut nv-qldpc-decoder trt_decoder; do - local name="examples/fpga/cpu_roce/$d" extra="" + local name="examples/fpga/cpu_roce/$(lane_label "$d")" extra="" [[ "$NO_FPGA" == true ]] && { skip_lane "$name" "--no-fpga"; continue; } case "$d" in nv-qldpc-decoder)