diff --git a/scripts/build_cudaq_with_realtime.sh b/scripts/build_cudaq_with_realtime.sh index f3c92205c..5725cc645 100755 --- a/scripts/build_cudaq_with_realtime.sh +++ b/scripts/build_cudaq_with_realtime.sh @@ -20,6 +20,16 @@ set -euo pipefail # Environment: # CUDAQ_SRC CUDA-Q source checkout, default: ./cudaq # CUDAQ_INSTALL_PREFIX CUDA-Q install prefix, default: /usr/local/cudaq +# CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS +# ON to build the hololink (HSB) bridge tools, default: OFF. +# When ON, HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR and +# HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR must point at a built +# holoscan-sensor-bridge checkout (see +# scripts/install_cudaq_with_realtime.sh). +# CUDAQ_REALTIME_BUILD_TESTS +# ON to build the realtime unit-test targets, default: OFF. +# (install_cudaq_with_realtime.sh may force this ON as a +# temporary workaround for older pinned CUDA-Q SHAs.) log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } @@ -32,16 +42,32 @@ export CXX : "${CUDAQ_SRC:=cudaq}" : "${CUDAQ_INSTALL_PREFIX:=/usr/local/cudaq}" +: "${CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS:=OFF}" +: "${CUDAQ_REALTIME_BUILD_TESTS:=OFF}" CUDAQ_SRC=$(cd "$CUDAQ_SRC" && pwd) -log "Building cudaq-realtime from $CUDAQ_SRC/realtime" +# When hololink tools are requested, the realtime CMake needs to know where the +# prebuilt holoscan-sensor-bridge lives so it can build the bridge-hololink +# wrapper library against it. +realtime_hololink_flags=() +if [ "$CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS" = "ON" ]; then + : "${HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR:?set HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR when hololink tools are enabled}" + : "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR:?set HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR when hololink tools are enabled}" + realtime_hololink_flags=( + "-DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=$HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR" + "-DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=$HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR" + ) +fi + +log "Building cudaq-realtime from $CUDAQ_SRC/realtime (hololink=$CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS, tests=$CUDAQ_REALTIME_BUILD_TESTS)" cmake -G Ninja -S "$CUDAQ_SRC/realtime" -B "$CUDAQ_SRC/realtime/build" \ -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \ -DCMAKE_INSTALL_PREFIX="$CUDAQ_INSTALL_PREFIX" \ - -DCUDAQ_REALTIME_BUILD_TESTS=OFF \ + -DCUDAQ_REALTIME_BUILD_TESTS="$CUDAQ_REALTIME_BUILD_TESTS" \ -DCUDAQ_REALTIME_BUILD_EXAMPLES=OFF \ - -DCUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS=OFF + -DCUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS="$CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS" \ + "${realtime_hololink_flags[@]}" cmake --build "$CUDAQ_SRC/realtime/build" --target install --parallel log "Building CUDA-Q with realtime support" diff --git a/scripts/install_cudaq_with_realtime.sh b/scripts/install_cudaq_with_realtime.sh index 8d0547a88..e09afe197 100755 --- a/scripts/install_cudaq_with_realtime.sh +++ b/scripts/install_cudaq_with_realtime.sh @@ -28,8 +28,23 @@ # CUDAQ_REPO/CUDAQ_REF override the .cudaq_version pin [from .cudaq_version] # FORCE_CHECKOUT=1 allow checkout over a dirty tree (DISCARDS changes) # +# Hololink / holoscan-sensor-bridge (HSB) support (built into cudaq-realtime so +# the QEC realtime libraries can drive the FPGA bridge). These deps are +# installed/built once per container; re-runs detect the prior install and skip +# the expensive clone+build: +# ENABLE_HOLOLINK=1 build DOCA/Holoscan/HSB + hololink tools [1] +# HSB_REPO/HSB_REF holoscan-sensor-bridge source pin +# HSB_ROOT/HSB_BUILD HSB checkout / build dir [/tmp/...] +# DOCA_VERSION DOCA repo version [3.3.0] +# DOCA_UBUNTU DOCA repo ubuntu flavor [ubuntu24.04] +# CUDA_NATIVE_ARCH GPU arch HSB is compiled for [80] +# HOLOLINK_REALTIME_TESTS +# realtime unit tests toggle; forced ON as a temporary +# workaround for the pinned CUDA-Q SHA (see block below) [ON] +# # Usage: # scripts/install_cudaq_with_realtime.sh +# ENABLE_HOLOLINK=0 scripts/install_cudaq_with_realtime.sh # skip HSB/hololink set -euo pipefail @@ -58,6 +73,44 @@ CC=${CC:-gcc} CXX=${CXX:-g++} FORCE_CHECKOUT=${FORCE_CHECKOUT:-0} +# Hololink / holoscan-sensor-bridge configuration. +ENABLE_HOLOLINK=${ENABLE_HOLOLINK:-1} +HSB_REPO=${HSB_REPO:-https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git} +HSB_REF=${HSB_REF:-2.6.0-EA2} +HSB_ROOT=${HSB_ROOT:-/tmp/holoscan-sensor-bridge} +HSB_BUILD=${HSB_BUILD:-$HSB_ROOT/build} +DOCA_VERSION=${DOCA_VERSION:-3.3.0} +DOCA_UBUNTU=${DOCA_UBUNTU:-ubuntu24.04} +CUDA_NATIVE_ARCH=${CUDA_NATIVE_ARCH:-80} + +# --------------------------------------------------------------------------- +# TEMPORARY WORKAROUND -- pinned CUDA-Q SHA predates the hololink cleanup. +# +# The CUDA-Q commit currently pinned in .cudaq_version still defines the +# `hololink_wrapper_generic` target under realtime/unittests/utils, so the +# hololink bridge library only links when the realtime unit tests are ALSO +# built. We therefore force CUDAQ_REALTIME_BUILD_TESTS=ON while hololink is +# enabled, otherwise the realtime build fails with: +# /usr/bin/ld: cannot find -lhololink_wrapper_generic +# +# >>> TO REMOVE THIS WORKAROUND once .cudaq_version is bumped to a CUDA-Q commit +# >>> that includes "[realtime] Remove redundant hololink_wrapper_generic static +# >>> library" (cuda-quantum@360f37681beaccbba276c785ca2325f674000b44): +# >>> 1. Delete this block and the CUDAQ_REALTIME_BUILD_TESTS entry added to +# >>> build_env below. +# >>> The hololink bridge then builds correctly with the realtime tests OFF. +# --------------------------------------------------------------------------- +HOLOLINK_REALTIME_TESTS=${HOLOLINK_REALTIME_TESTS:-ON} + +# apt/dpkg/curl-to-system-dirs need root; use sudo when not already root. +if [ "$(id -u)" -eq 0 ]; then + SUDO="" +elif command -v sudo >/dev/null 2>&1; then + SUDO="sudo" +else + SUDO="" +fi + command -v jq >/dev/null 2>&1 || die "jq is required (parses .cudaq_version)" command -v git >/dev/null 2>&1 || die "git is required" command -v cmake >/dev/null 2>&1 || die "cmake is required" @@ -106,6 +159,115 @@ fi [ -x "$LLVM_INSTALL_PREFIX/bin/llvm-config" ] || \ log "WARNING: no llvm-config at $LLVM_INSTALL_PREFIX/bin; set LLVM_INSTALL_PREFIX if the build can't find LLVM." +# --------------------------------------------------------------------------- +# 1.5 Install DOCA / Holoscan SDK / holoscan-sensor-bridge (once per container). +# +# cudaq-realtime's hololink tools link against DOCA GPUNetIO, the Holoscan +# SDK, and a locally-built holoscan-sensor-bridge (HSB). These are heavy to +# install/build, so each step is guarded and skipped when already present. +# Mirrors .github/actions/build-lib/build_qec.sh but made idempotent. +# +# NOTE: the cudaqx CI container ships Mellanox OFED pre-installed, so we must +# NOT run doca's install_dev_prerequisites.sh / doca-all (it conflicts with +# the container OFED). We install only the GPUNetIO dev headers we need. +# --------------------------------------------------------------------------- + +# True when HSB has already been built (its libs exist under $HSB_BUILD). +hsb_built() { + [ -d "$HSB_BUILD" ] && \ + find "$HSB_BUILD" -name 'libhololink_core*' -print -quit 2>/dev/null | grep -q . +} + +install_hsb_deps() { + command -v nvcc >/dev/null 2>&1 || \ + die "nvcc not found; DOCA/Holoscan/HSB build requires the CUDA toolkit on PATH" + + local cuda_major cuda_full cuda_dash doca_arch doca_repo + cuda_major=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p') + cuda_full=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p') + cuda_dash=$(echo "$cuda_full" | sed 's/\./-/') + + log "Installing build tools (ninja, curl, pkg-config)" + $SUDO apt-get update + $SUDO apt-get install -y --no-install-recommends ninja-build curl pkg-config + + # --- DOCA GPUNetIO dev headers --- + if [ -d /opt/mellanox/doca/include ]; then + log "DOCA already present at /opt/mellanox/doca; skipping" + else + log "Installing DOCA GPUNetIO dev package (DOCA $DOCA_VERSION / $DOCA_UBUNTU)" + doca_arch=$(uname -m) + case "$doca_arch" in aarch64|arm64) doca_arch="arm64-sbsa" ;; esac + doca_repo="https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}/${DOCA_UBUNTU}/$doca_arch" + $SUDO curl -fsSL "$doca_repo/GPG-KEY-Mellanox.pub" \ + -o /usr/share/keyrings/GPG-KEY-Mellanox.pub + echo "deb [signed-by=/usr/share/keyrings/GPG-KEY-Mellanox.pub] $doca_repo /" \ + | $SUDO tee /etc/apt/sources.list.d/doca.list >/dev/null + $SUDO apt-get update + $SUDO apt-get install -y --no-install-recommends libdoca-sdk-gpunetio-dev + fi + + # hololink_core links CUDA::nvrtc -- must match the exact toolkit version. + $SUDO apt-get install -y "cuda-nvrtc-dev-$cuda_dash" 2>/dev/null || true + + # --- Holoscan SDK --- + if [ -d /opt/nvidia/holoscan ]; then + log "Holoscan SDK already present at /opt/nvidia/holoscan; skipping" + else + log "Installing Holoscan SDK (cuda $cuda_major)" + $SUDO apt-get install -y --no-install-recommends "holoscan-cuda-$cuda_major" || { + # Force-install if normal install fails due to missing (OFED) deps. + local hsdk_tmp + hsdk_tmp=$(mktemp -d) + ( cd "$hsdk_tmp" \ + && apt-get download holoscan "holoscan-cuda-$cuda_major" \ + && $SUDO dpkg --force-depends -i holoscan*.deb ) + rm -rf "$hsdk_tmp" + } + fi + + [ -d /opt/mellanox/doca/include ] || die "DOCA SDK installation failed" + [ -d /opt/nvidia/holoscan ] || die "Holoscan SDK installation failed" + + # --- holoscan-sensor-bridge (hololink) --- + if hsb_built; then + log "holoscan-sensor-bridge already built at $HSB_BUILD; skipping clone+build" + else + log "Building holoscan-sensor-bridge $HSB_REF at $HSB_ROOT" + rm -rf "$HSB_ROOT" + git clone --depth 1 --branch "$HSB_REF" "$HSB_REPO" "$HSB_ROOT" + ( + cd "$HSB_ROOT" + # Strip operators we don't need to avoid configure failures from missing deps. + sed -i '/add_subdirectory(audio_packetizer)/d; /add_subdirectory(compute_crc)/d; + /add_subdirectory(csi_to_bayer)/d; /add_subdirectory(image_processor)/d; + /add_subdirectory(iq_dec)/d; /add_subdirectory(iq_enc)/d; + /add_subdirectory(linux_coe_receiver)/d; /add_subdirectory(linux_receiver)/d; + /add_subdirectory(packed_format_converter)/d; /add_subdirectory(sub_frame_combiner)/d; + /add_subdirectory(udp_transmitter)/d; /add_subdirectory(emulator)/d; + /add_subdirectory(sig_gen)/d; /add_subdirectory(sig_viewer)/d' \ + src/hololink/operators/CMakeLists.txt + CUDA_NATIVE_ARCH="$CUDA_NATIVE_ARCH" cmake -G Ninja -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DHOLOLINK_BUILD_ONLY_NATIVE=OFF \ + -DHOLOLINK_BUILD_PYTHON=OFF \ + -DHOLOLINK_BUILD_TESTS=OFF \ + -DHOLOLINK_BUILD_TOOLS=OFF \ + -DHOLOLINK_BUILD_EXAMPLES=OFF \ + -DHOLOLINK_BUILD_EMULATOR=OFF + cmake --build build --target gpu_roce_transceiver hololink_core + ) + hsb_built || die "holoscan-sensor-bridge build finished but libhololink_core is missing under $HSB_BUILD" + log "holoscan-sensor-bridge built at $HSB_BUILD" + fi +} + +if [ "$ENABLE_HOLOLINK" = "1" ]; then + install_hsb_deps +else + log "ENABLE_HOLOLINK=0 -- skipping DOCA/Holoscan/HSB; building realtime without hololink tools" +fi + # --------------------------------------------------------------------------- # 2. Clean the build dirs, then build + install realtime AND the full CUDA-Q # (with device_call) by delegating to the canonical CI recipe instead of @@ -122,10 +284,25 @@ log "Cleaning build directories" rm -rf "$CUDAQ_SRC/realtime/build" "$CUDAQ_SRC/build" log "Building realtime + CUDA-Q via $build_script" -CUDAQ_SRC="$CUDAQ_SRC" \ -CUDAQ_INSTALL_PREFIX="$CUDAQ_INSTALL_PREFIX" \ -LLVM_INSTALL_PREFIX="$LLVM_INSTALL_PREFIX" \ - bash "$build_script" "$BUILD_TYPE" "" "$CC" "$CXX" +build_env=( + "CUDAQ_SRC=$CUDAQ_SRC" + "CUDAQ_INSTALL_PREFIX=$CUDAQ_INSTALL_PREFIX" + "LLVM_INSTALL_PREFIX=$LLVM_INSTALL_PREFIX" +) +if [ "$ENABLE_HOLOLINK" = "1" ]; then + # Turn on the hololink bridge tools in the realtime build and point them at + # the HSB checkout we installed/built above. + build_env+=( + "CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS=ON" + "HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=$HSB_ROOT" + "HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=$HSB_BUILD" + # TEMPORARY: required by the pinned CUDA-Q SHA -- see the HOLOLINK_REALTIME_TESTS + # workaround block near the top of this script. Remove once .cudaq_version is + # bumped past the hololink_wrapper_generic cleanup. + "CUDAQ_REALTIME_BUILD_TESTS=$HOLOLINK_REALTIME_TESTS" + ) +fi +env "${build_env[@]}" bash "$build_script" "$BUILD_TYPE" "" "$CC" "$CXX" # --------------------------------------------------------------------------- # 3. Verify device_call support actually installed (the markers from manual @@ -140,4 +317,37 @@ if [ -f "$dc_hdr" ]; then echo " [ok] $dc_hdr"; else echo " [MISSING] $dc [ "$ok" = "1" ] || die "build finished but device_call artifacts are missing -- check that CUDA was found and CUDAQ_ENABLE_REALTIME was TRUE." log "Done. CUDA-Q with device_call support installed at $CUDAQ_INSTALL_PREFIX" -log "Configure cudaqx with: -DCUDAQ_DIR=$CUDAQ_INSTALL_PREFIX/lib/cmake/cudaq -DCUDAQ_REALTIME_ROOT=$CUDAQ_INSTALL_PREFIX" + +# --------------------------------------------------------------------------- +# 4. Print a copy-pasteable cmake command for configuring cudaqx. +# (When hololink is enabled we include the HOLOSCAN_SENSOR_BRIDGE flags so +# the GPU-RoCE decoding server is built instead of silently disabled.) +# --------------------------------------------------------------------------- +printf '\n' +log "Next step -- configure cudaqx (from a build dir under the repo root):" +if [ "$ENABLE_HOLOLINK" = "1" ]; then + cat <