Skip to content

Update dev scripts to install cudaq-realtime w/ HSB #693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bmhowe23 wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Update dev scripts to install cudaq-realtime w/ HSB #693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fa6627e
Update dev scripts to install cudaq-realtime w/ HSB
bmhowe23 Jul 14, 2026
74c8a1e
Merge branch 'main' into pr-update-dev-scripts
bmhowe23 Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions scripts/build_cudaq_with_realtime.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,16 @@ set -euo pipefail
# Environment:
# CUDAQ_SRC CUDA-Q source checkout, default: ./cudaq
# CUDAQ_INSTALL_PREFIX CUDA-Q install prefix, default: /usr/local/cudaq
# CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS
# ON to build the hololink (HSB) bridge tools, default: OFF.
# When ON, HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR and
# HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR must point at a built
# holoscan-sensor-bridge checkout (see
# scripts/install_cudaq_with_realtime.sh).
# CUDAQ_REALTIME_BUILD_TESTS
# ON to build the realtime unit-test targets, default: OFF.
# (install_cudaq_with_realtime.sh may force this ON as a
# temporary workaround for older pinned CUDA-Q SHAs.)

log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; }

Expand All @@ -32,16 +42,32 @@ export CXX

: "${CUDAQ_SRC:=cudaq}"
: "${CUDAQ_INSTALL_PREFIX:=/usr/local/cudaq}"
: "${CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS:=OFF}"
: "${CUDAQ_REALTIME_BUILD_TESTS:=OFF}"

CUDAQ_SRC=$(cd "$CUDAQ_SRC" && pwd)

log "Building cudaq-realtime from $CUDAQ_SRC/realtime"
# When hololink tools are requested, the realtime CMake needs to know where the
# prebuilt holoscan-sensor-bridge lives so it can build the bridge-hololink
# wrapper library against it.
realtime_hololink_flags=()
if [ "$CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS" = "ON" ]; then
: "${HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR:?set HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR when hololink tools are enabled}"
: "${HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR:?set HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR when hololink tools are enabled}"
realtime_hololink_flags=(
"-DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=$HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR"
"-DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=$HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR"
)
fi

log "Building cudaq-realtime from $CUDAQ_SRC/realtime (hololink=$CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS, tests=$CUDAQ_REALTIME_BUILD_TESTS)"
cmake -G Ninja -S "$CUDAQ_SRC/realtime" -B "$CUDAQ_SRC/realtime/build" \
-DCMAKE_BUILD_TYPE="$BUILD_TYPE" \
-DCMAKE_INSTALL_PREFIX="$CUDAQ_INSTALL_PREFIX" \
-DCUDAQ_REALTIME_BUILD_TESTS=OFF \
-DCUDAQ_REALTIME_BUILD_TESTS="$CUDAQ_REALTIME_BUILD_TESTS" \
-DCUDAQ_REALTIME_BUILD_EXAMPLES=OFF \
-DCUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS=OFF
-DCUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS="$CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS" \
"${realtime_hololink_flags[@]}"
cmake --build "$CUDAQ_SRC/realtime/build" --target install --parallel

log "Building CUDA-Q with realtime support"
Expand Down
220 changes: 215 additions & 5 deletions scripts/install_cudaq_with_realtime.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,23 @@
# CUDAQ_REPO/CUDAQ_REF override the .cudaq_version pin [from .cudaq_version]
# FORCE_CHECKOUT=1 allow checkout over a dirty tree (DISCARDS changes)
#
# Hololink / holoscan-sensor-bridge (HSB) support (built into cudaq-realtime so
# the QEC realtime libraries can drive the FPGA bridge). These deps are
# installed/built once per container; re-runs detect the prior install and skip
# the expensive clone+build:
# ENABLE_HOLOLINK=1 build DOCA/Holoscan/HSB + hololink tools [1]
# HSB_REPO/HSB_REF holoscan-sensor-bridge source pin
# HSB_ROOT/HSB_BUILD HSB checkout / build dir [/tmp/...]
# DOCA_VERSION DOCA repo version [3.3.0]
# DOCA_UBUNTU DOCA repo ubuntu flavor [ubuntu24.04]
# CUDA_NATIVE_ARCH GPU arch HSB is compiled for [80]
# HOLOLINK_REALTIME_TESTS
# realtime unit tests toggle; forced ON as a temporary
# workaround for the pinned CUDA-Q SHA (see block below) [ON]
#
# Usage:
# scripts/install_cudaq_with_realtime.sh
# ENABLE_HOLOLINK=0 scripts/install_cudaq_with_realtime.sh # skip HSB/hololink

set -euo pipefail

Expand Down Expand Up @@ -58,6 +73,44 @@ CC=${CC:-gcc}
CXX=${CXX:-g++}
FORCE_CHECKOUT=${FORCE_CHECKOUT:-0}

# Hololink / holoscan-sensor-bridge configuration.
ENABLE_HOLOLINK=${ENABLE_HOLOLINK:-1}
HSB_REPO=${HSB_REPO:-https://github.com/nvidia-holoscan/holoscan-sensor-bridge.git}
HSB_REF=${HSB_REF:-2.6.0-EA2}
HSB_ROOT=${HSB_ROOT:-/tmp/holoscan-sensor-bridge}
HSB_BUILD=${HSB_BUILD:-$HSB_ROOT/build}
DOCA_VERSION=${DOCA_VERSION:-3.3.0}
DOCA_UBUNTU=${DOCA_UBUNTU:-ubuntu24.04}
CUDA_NATIVE_ARCH=${CUDA_NATIVE_ARCH:-80}

# ---------------------------------------------------------------------------
# TEMPORARY WORKAROUND -- pinned CUDA-Q SHA predates the hololink cleanup.
#
# The CUDA-Q commit currently pinned in .cudaq_version still defines the
# `hololink_wrapper_generic` target under realtime/unittests/utils, so the
# hololink bridge library only links when the realtime unit tests are ALSO
# built. We therefore force CUDAQ_REALTIME_BUILD_TESTS=ON while hololink is
# enabled, otherwise the realtime build fails with:
# /usr/bin/ld: cannot find -lhololink_wrapper_generic
#
# >>> TO REMOVE THIS WORKAROUND once .cudaq_version is bumped to a CUDA-Q commit
# >>> that includes "[realtime] Remove redundant hololink_wrapper_generic static
# >>> library" (cuda-quantum@360f37681beaccbba276c785ca2325f674000b44):
# >>> 1. Delete this block and the CUDAQ_REALTIME_BUILD_TESTS entry added to
# >>> build_env below.
# >>> The hololink bridge then builds correctly with the realtime tests OFF.
# ---------------------------------------------------------------------------
HOLOLINK_REALTIME_TESTS=${HOLOLINK_REALTIME_TESTS:-ON}

# apt/dpkg/curl-to-system-dirs need root; use sudo when not already root.
if [ "$(id -u)" -eq 0 ]; then
SUDO=""
elif command -v sudo >/dev/null 2>&1; then
SUDO="sudo"
else
SUDO=""
fi

command -v jq >/dev/null 2>&1 || die "jq is required (parses .cudaq_version)"
command -v git >/dev/null 2>&1 || die "git is required"
command -v cmake >/dev/null 2>&1 || die "cmake is required"
Expand Down Expand Up @@ -106,6 +159,115 @@ fi
[ -x "$LLVM_INSTALL_PREFIX/bin/llvm-config" ] || \
log "WARNING: no llvm-config at $LLVM_INSTALL_PREFIX/bin; set LLVM_INSTALL_PREFIX if the build can't find LLVM."

# ---------------------------------------------------------------------------
# 1.5 Install DOCA / Holoscan SDK / holoscan-sensor-bridge (once per container).
#
# cudaq-realtime's hololink tools link against DOCA GPUNetIO, the Holoscan
# SDK, and a locally-built holoscan-sensor-bridge (HSB). These are heavy to
# install/build, so each step is guarded and skipped when already present.
# Mirrors .github/actions/build-lib/build_qec.sh but made idempotent.
#
# NOTE: the cudaqx CI container ships Mellanox OFED pre-installed, so we must
# NOT run doca's install_dev_prerequisites.sh / doca-all (it conflicts with
# the container OFED). We install only the GPUNetIO dev headers we need.
# ---------------------------------------------------------------------------

# True when HSB has already been built (its libs exist under $HSB_BUILD).
hsb_built() {
[ -d "$HSB_BUILD" ] && \
find "$HSB_BUILD" -name 'libhololink_core*' -print -quit 2>/dev/null | grep -q .
}

install_hsb_deps() {
command -v nvcc >/dev/null 2>&1 || \
die "nvcc not found; DOCA/Holoscan/HSB build requires the CUDA toolkit on PATH"

local cuda_major cuda_full cuda_dash doca_arch doca_repo
cuda_major=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\).*$/\1/p')
cuda_full=$(nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p')
cuda_dash=$(echo "$cuda_full" | sed 's/\./-/')

log "Installing build tools (ninja, curl, pkg-config)"
$SUDO apt-get update
$SUDO apt-get install -y --no-install-recommends ninja-build curl pkg-config

# --- DOCA GPUNetIO dev headers ---
if [ -d /opt/mellanox/doca/include ]; then
log "DOCA already present at /opt/mellanox/doca; skipping"
else
log "Installing DOCA GPUNetIO dev package (DOCA $DOCA_VERSION / $DOCA_UBUNTU)"
doca_arch=$(uname -m)
case "$doca_arch" in aarch64|arm64) doca_arch="arm64-sbsa" ;; esac
doca_repo="https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}/${DOCA_UBUNTU}/$doca_arch"
$SUDO curl -fsSL "$doca_repo/GPG-KEY-Mellanox.pub" \
-o /usr/share/keyrings/GPG-KEY-Mellanox.pub
echo "deb [signed-by=/usr/share/keyrings/GPG-KEY-Mellanox.pub] $doca_repo /" \
| $SUDO tee /etc/apt/sources.list.d/doca.list >/dev/null
$SUDO apt-get update
$SUDO apt-get install -y --no-install-recommends libdoca-sdk-gpunetio-dev
fi

# hololink_core links CUDA::nvrtc -- must match the exact toolkit version.
$SUDO apt-get install -y "cuda-nvrtc-dev-$cuda_dash" 2>/dev/null || true

# --- Holoscan SDK ---
if [ -d /opt/nvidia/holoscan ]; then
log "Holoscan SDK already present at /opt/nvidia/holoscan; skipping"
else
log "Installing Holoscan SDK (cuda $cuda_major)"
$SUDO apt-get install -y --no-install-recommends "holoscan-cuda-$cuda_major" || {
# Force-install if normal install fails due to missing (OFED) deps.
local hsdk_tmp
hsdk_tmp=$(mktemp -d)
( cd "$hsdk_tmp" \
&& apt-get download holoscan "holoscan-cuda-$cuda_major" \
&& $SUDO dpkg --force-depends -i holoscan*.deb )
rm -rf "$hsdk_tmp"
}
fi

[ -d /opt/mellanox/doca/include ] || die "DOCA SDK installation failed"
[ -d /opt/nvidia/holoscan ] || die "Holoscan SDK installation failed"

# --- holoscan-sensor-bridge (hololink) ---
if hsb_built; then
log "holoscan-sensor-bridge already built at $HSB_BUILD; skipping clone+build"
else
log "Building holoscan-sensor-bridge $HSB_REF at $HSB_ROOT"
rm -rf "$HSB_ROOT"
git clone --depth 1 --branch "$HSB_REF" "$HSB_REPO" "$HSB_ROOT"
(
cd "$HSB_ROOT"
# Strip operators we don't need to avoid configure failures from missing deps.
sed -i '/add_subdirectory(audio_packetizer)/d; /add_subdirectory(compute_crc)/d;
/add_subdirectory(csi_to_bayer)/d; /add_subdirectory(image_processor)/d;
/add_subdirectory(iq_dec)/d; /add_subdirectory(iq_enc)/d;
/add_subdirectory(linux_coe_receiver)/d; /add_subdirectory(linux_receiver)/d;
/add_subdirectory(packed_format_converter)/d; /add_subdirectory(sub_frame_combiner)/d;
/add_subdirectory(udp_transmitter)/d; /add_subdirectory(emulator)/d;
/add_subdirectory(sig_gen)/d; /add_subdirectory(sig_viewer)/d' \
src/hololink/operators/CMakeLists.txt
CUDA_NATIVE_ARCH="$CUDA_NATIVE_ARCH" cmake -G Ninja -S . -B build \
-DCMAKE_BUILD_TYPE=Release \
-DHOLOLINK_BUILD_ONLY_NATIVE=OFF \
-DHOLOLINK_BUILD_PYTHON=OFF \
-DHOLOLINK_BUILD_TESTS=OFF \
-DHOLOLINK_BUILD_TOOLS=OFF \
-DHOLOLINK_BUILD_EXAMPLES=OFF \
-DHOLOLINK_BUILD_EMULATOR=OFF
cmake --build build --target gpu_roce_transceiver hololink_core
)
hsb_built || die "holoscan-sensor-bridge build finished but libhololink_core is missing under $HSB_BUILD"
log "holoscan-sensor-bridge built at $HSB_BUILD"
fi
}

if [ "$ENABLE_HOLOLINK" = "1" ]; then
install_hsb_deps
else
log "ENABLE_HOLOLINK=0 -- skipping DOCA/Holoscan/HSB; building realtime without hololink tools"
fi

# ---------------------------------------------------------------------------
# 2. Clean the build dirs, then build + install realtime AND the full CUDA-Q
# (with device_call) by delegating to the canonical CI recipe instead of
Expand All @@ -122,10 +284,25 @@ log "Cleaning build directories"
rm -rf "$CUDAQ_SRC/realtime/build" "$CUDAQ_SRC/build"

log "Building realtime + CUDA-Q via $build_script"
CUDAQ_SRC="$CUDAQ_SRC" \
CUDAQ_INSTALL_PREFIX="$CUDAQ_INSTALL_PREFIX" \
LLVM_INSTALL_PREFIX="$LLVM_INSTALL_PREFIX" \
bash "$build_script" "$BUILD_TYPE" "" "$CC" "$CXX"
build_env=(
"CUDAQ_SRC=$CUDAQ_SRC"
"CUDAQ_INSTALL_PREFIX=$CUDAQ_INSTALL_PREFIX"
"LLVM_INSTALL_PREFIX=$LLVM_INSTALL_PREFIX"
)
if [ "$ENABLE_HOLOLINK" = "1" ]; then
# Turn on the hololink bridge tools in the realtime build and point them at
# the HSB checkout we installed/built above.
build_env+=(
"CUDAQ_REALTIME_ENABLE_HOLOLINK_TOOLS=ON"
"HOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR=$HSB_ROOT"
"HOLOSCAN_SENSOR_BRIDGE_BUILD_DIR=$HSB_BUILD"
# TEMPORARY: required by the pinned CUDA-Q SHA -- see the HOLOLINK_REALTIME_TESTS
# workaround block near the top of this script. Remove once .cudaq_version is
# bumped past the hololink_wrapper_generic cleanup.
"CUDAQ_REALTIME_BUILD_TESTS=$HOLOLINK_REALTIME_TESTS"
)
fi
env "${build_env[@]}" bash "$build_script" "$BUILD_TYPE" "" "$CC" "$CXX"

# ---------------------------------------------------------------------------
# 3. Verify device_call support actually installed (the markers from manual
Expand All @@ -140,4 +317,37 @@ if [ -f "$dc_hdr" ]; then echo " [ok] $dc_hdr"; else echo " [MISSING] $dc
[ "$ok" = "1" ] || die "build finished but device_call artifacts are missing -- check that CUDA was found and CUDAQ_ENABLE_REALTIME was TRUE."

log "Done. CUDA-Q with device_call support installed at $CUDAQ_INSTALL_PREFIX"
log "Configure cudaqx with: -DCUDAQ_DIR=$CUDAQ_INSTALL_PREFIX/lib/cmake/cudaq -DCUDAQ_REALTIME_ROOT=$CUDAQ_INSTALL_PREFIX"

# ---------------------------------------------------------------------------
# 4. Print a copy-pasteable cmake command for configuring cudaqx.
# (When hololink is enabled we include the HOLOSCAN_SENSOR_BRIDGE flags so
# the GPU-RoCE decoding server is built instead of silently disabled.)
# ---------------------------------------------------------------------------
printf '\n'
log "Next step -- configure cudaqx (from a build dir under the repo root):"
if [ "$ENABLE_HOLOLINK" = "1" ]; then
cat <<EOF

mkdir -p "$repo_root/build" && cd "$repo_root/build"
cmake -G Ninja \\
-DCUDAQ_REALTIME_ROOT="$CUDAQ_INSTALL_PREFIX" \\
-DCUDAQ_DIR="$CUDAQ_INSTALL_PREFIX/lib/cmake/cudaq" \\
-DCUDAQX_QEC_ENABLE_HOLOLINK_TOOLS=ON \\
-DHOLOSCAN_SENSOR_BRIDGE_SOURCE_DIR="$HSB_ROOT" \\
-DHOLOSCAN_SENSOR_BRIDGE_BUILD_DIR="$HSB_BUILD" \\
..
cmake --build .

EOF
else
cat <<EOF

mkdir -p "$repo_root/build" && cd "$repo_root/build"
cmake -G Ninja \\
-DCUDAQ_REALTIME_ROOT="$CUDAQ_INSTALL_PREFIX" \\
-DCUDAQ_DIR="$CUDAQ_INSTALL_PREFIX/lib/cmake/cudaq" \\
..
cmake --build .

EOF
fi
Loading