diff --git a/.github/workflows/build-and-measure.yml b/.github/workflows/build-and-measure.yml index 50b85a6..5bd8716 100644 --- a/.github/workflows/build-and-measure.yml +++ b/.github/workflows/build-and-measure.yml @@ -21,8 +21,8 @@ jobs: build: name: build (latrec=${{ matrix.latrec }}) runs-on: ubuntu-latest - # Cold, this is asn1c-from-source + libe3 + jbpf and its third-party tree + - # the controller. Comfortably inside an hour; nowhere near the 6h default. + # Cold, this is asn1c-from-source + libe3 + ocudu's jbpf and its third-party + # tree + the controller. Comfortably inside an hour; nowhere near the 6h default. timeout-minutes: 60 strategy: fail-fast: false @@ -32,11 +32,25 @@ jobs: steps: - uses: actions/checkout@v4 with: - # build.sh fetches the submodules itself and then runs jbpf's - # init_and_patch_submodules.sh over them. Letting the checkout action - # do it first would hand that patch script a tree it did not lay out. + # build.sh fetches the libe3 submodule itself. submodules: false + # jbpf is not vendored here: the controller is built against the gNB's own + # jbpf, ocudu's external/jbpf, so the two ends of the IPC are always the + # same jbpf. CI takes it from the public ocudu mirror. Only that submodule + # is fetched; the rest of ocudu is not needed to build the controller. + - name: Check out ocudu (for its jbpf and hook contract) + uses: actions/checkout@v4 + with: + repository: wineslab/ocudu-e3 + ref: e3 + path: .deps/ocudu-e3 + fetch-depth: 1 + submodules: false + + - name: Populate ocudu's jbpf + run: git -C .deps/ocudu-e3 submodule update --init --recursive --depth 1 external/jbpf + # The libe3 pin decides which asn1c revision is wanted, so it belongs in # the cache key. hashFiles cannot read a gitlink, and the submodule is not # checked out yet, so read the pinned SHA straight out of the tree object. @@ -69,6 +83,7 @@ jobs: # which breaks both ccache reuse across heterogeneous runners and any # comparison of the measurements below between runs. E3C_CMAKE_ARGS: -DUSE_NATIVE=OFF + OCUDU_DIR: ${{ github.workspace }}/.deps/ocudu-e3 run: | sudo apt-get update sudo apt-get install -y --no-install-recommends ccache @@ -89,6 +104,7 @@ jobs: echo "nproc: $(nproc)" echo "/dev/shm: $(df -h /dev/shm | tail -1)" echo "libe3 pin: $(git -C libe3 rev-parse --short HEAD) ($(cat libe3/VERSION))" + echo "ocudu-e3: $(git -C .deps/ocudu-e3 rev-parse --short HEAD), jbpf pin: $(git -C .deps/ocudu-e3 rev-parse --short HEAD:external/jbpf)" echo '```' } > report.md diff --git a/.gitmodules b/.gitmodules index df6e9f6..335c945 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "jbpf"] - path = jbpf - url = https://github.com/microsoft/jbpf.git [submodule "libe3"] path = libe3 url = https://github.com/wineslab/libe3.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b7fff8..a7fa7f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,33 +25,64 @@ file(MAKE_DIRECTORY ${OUTPUT_DIR}) file(MAKE_DIRECTORY ${OUTPUT_DIR}/inc) file(MAKE_DIRECTORY ${OUTPUT_DIR}/lib) -# ---- Build jbpf (full build) ---- +# ---- ocudu checkout ---- +# The gNB this controller talks to is built from an ocudu checkout, and two things +# come from it: the jbpf tree (below) and the hook context contract the codelets +# and the verifier read (include/ocudu/janus). Clone it next to this repository, +# or point OCUDU_DIR at it. +set(OCUDU_DIR "${CMAKE_SOURCE_DIR}/../ocudu-e3" + CACHE PATH "Path to the ocudu checkout the gNB is built from") +get_filename_component(OCUDU_DIR "${OCUDU_DIR}" ABSOLUTE) + +# ---- jbpf: the gNB's own tree ---- +# The controller is the jbpf IPC primary and the gNB the secondary. Both map one +# shared region and exchange one channel and message format, so they must be +# built from the SAME jbpf. Instead of pinning a second copy here and keeping the +# two pins, and their patches, in step by hand, build against ocudu's +# external/jbpf: the tree the gNB is built from. Its patches are ocudu's too +# (ocudu apply_jbpf_patches.sh; build.sh runs it). +# +# Pass -DJBPF_DIR only to build against a different jbpf on purpose. +if(NOT JBPF_DIR) + set(JBPF_DIR "${OCUDU_DIR}/external/jbpf") +endif() +get_filename_component(JBPF_DIR "${JBPF_DIR}" ABSOLUTE) + +if(NOT EXISTS "${JBPF_DIR}/CMakeLists.txt") + message(FATAL_ERROR + "jbpf not found at JBPF_DIR='${JBPF_DIR}'.\n" + "The controller builds against the gNB's jbpf (ocudu's external/jbpf). " + "Clone ocudu next to this repository, or pass -DOCUDU_DIR=/path/to/ocudu, " + "then populate and patch it:\n" + " git -C submodule update --init --recursive external/jbpf\n" + " /apply_jbpf_patches.sh") +endif() + # Disable tests and examples — we only need the libraries set(BUILD_TESTING OFF CACHE BOOL "Disable jbpf tests" FORCE) # Stop jbpf from re-initialising its own submodules. # # jbpf's CMakeLists.txt runs init_and_patch_submodules.sh when INITIALIZE_SUBMODULES -# is ON, which is its default. That script `rm -rf`s and re-clones jbpf/3p/*, so on -# any tree where git cannot resolve the submodules — a tarball, an `oc cp`/rsync -# copy into a container, a git worktree — it DESTROYS the working 3p checkout and -# then fails: -# -# fatal: not a git repository -# cp: cannot create regular file '3p/ebpf-verifier/': Not a directory -# -# and the tree is left with ebpf-verifier and ubpf gone, so a plain re-configure -# cannot recover. build.sh already populates the submodules itself -# (jbpf/init_and_patch_submodules.sh), so there is nothing to gain from letting -# jbpf redo it. ocudu forces the same value for the same reason +# is ON, which is its default. That script `rm -rf`s and re-clones 3p/*, so on any +# tree where git cannot resolve the submodules — a tarball, an `oc cp`/rsync copy +# into a container — it DESTROYS the working 3p checkout and then fails. The tree +# is ocudu's, populated and patched on ocudu's side, so there is nothing to gain +# from letting jbpf redo it. ocudu forces the same value for the same reason # (ocudu CMakeLists.txt, the ENABLE_JBPF block). # # FORCE because it is a cached option in jbpf; without FORCE a stale ON from an # earlier configure would survive. set(INITIALIZE_SUBMODULES OFF CACHE BOOL "Let jbpf init its own submodules" FORCE) -# Fail early and legibly if the submodules are genuinely absent, rather than -# letting the first missing header surface hundreds of lines into the build. +# Build jbpf with the options ocudu builds it with (ocudu CMakeLists.txt, the +# ENABLE_JBPF block), so both ends of the IPC are the same jbpf in every respect, +# not just the same commit. +set(JBPF_STATIC ON CACHE BOOL "Build jbpf as static library" FORCE) +set(JBPF_EXPERIMENTAL_FEATURES ON CACHE BOOL "Enable jbpf experimental features" FORCE) + +# Fail early and legibly if the 3p submodules are absent, rather than letting the +# first missing header surface hundreds of lines into the build. # # A per-module sentinel FILE, one that module actually ships. # @@ -67,26 +98,73 @@ set(_jbpf_3p_checks "mimalloc/CMakeLists.txt" "ck/configure") foreach(_jbpf_3p_file IN LISTS _jbpf_3p_checks) - if(NOT EXISTS "${CMAKE_SOURCE_DIR}/jbpf/3p/${_jbpf_3p_file}") + if(NOT EXISTS "${JBPF_DIR}/3p/${_jbpf_3p_file}") message(FATAL_ERROR - "jbpf/3p/${_jbpf_3p_file} is missing, so jbpf's third-party submodules are " - "not populated. jbpf's own auto-init is disabled here (see above, it would " - "destroy them), so populate them yourself:\n" - " git submodule update --init --recursive # then\n" - " ./jbpf/init_and_patch_submodules.sh\n" - "or, for a copied tree with no git metadata, copy jbpf/ wholesale from a " - "working checkout.") + "${JBPF_DIR}/3p/${_jbpf_3p_file} is missing, so jbpf's third-party " + "submodules are not populated. jbpf's own auto-init is disabled here (see " + "above, it would destroy them), so populate them on the ocudu side:\n" + " git -C ${OCUDU_DIR} submodule update --init --recursive external/jbpf\n" + " ${OCUDU_DIR}/apply_jbpf_patches.sh") endif() endforeach() unset(_jbpf_3p_checks) +# Say which jbpf this is, and whether it is the one ocudu pins. A jbpf checked +# out at another commit than ocudu's gitlink would build a gNB and a controller +# that disagree, so warn rather than let that surface as an IPC failure. +execute_process(COMMAND git -C "${JBPF_DIR}" rev-parse HEAD + OUTPUT_VARIABLE _jbpf_rev OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET RESULT_VARIABLE _jbpf_rev_rc) +execute_process(COMMAND git -C "${OCUDU_DIR}" rev-parse HEAD:external/jbpf + OUTPUT_VARIABLE _jbpf_pin OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET RESULT_VARIABLE _jbpf_pin_rc) +if(NOT _jbpf_rev_rc EQUAL 0) + set(_jbpf_rev "unknown (not a git checkout)") +endif() +if(_jbpf_rev_rc EQUAL 0 AND _jbpf_pin_rc EQUAL 0 AND NOT _jbpf_rev STREQUAL _jbpf_pin) + message(WARNING + "jbpf at ${JBPF_DIR} is at ${_jbpf_rev}, but ocudu pins external/jbpf at " + "${_jbpf_pin}. The gNB and the controller must be built from the same jbpf:\n" + " git -C ${OCUDU_DIR} submodule update --init --recursive external/jbpf") +endif() + +# The gNB stamps gnb_ts_ns with CLOCK_MONOTONIC, and the codelet's stamps come from +# jbpf_time_get_ns(), which is only monotonic with ocudu's +# jbpf_patches/jbpf_monotonic_time.patch applied. The controller does not run +# codelets, so this build does not need it, but a gNB built from this same tree +# would produce stage intervals that mix two clocks. Upstream jbpf calls +# clock_gettime exactly once in that file, with CLOCK_REALTIME. +file(STRINGS "${JBPF_DIR}/src/core/jbpf_helper_impl.c" _jbpf_mono + REGEX "clock_gettime\\(CLOCK_MONOTONIC") +if(_jbpf_mono) + set(_jbpf_clock "monotonic patch applied") +else() + set(_jbpf_clock "monotonic patch NOT applied") + message(WARNING + "jbpf at ${JBPF_DIR} lacks jbpf_monotonic_time.patch, so a gNB built from it " + "stamps codelet_ts_ns with CLOCK_REALTIME while gnb_ts_ns is CLOCK_MONOTONIC. " + "Apply it on the ocudu side: ${OCUDU_DIR}/apply_jbpf_patches.sh") +endif() +message(STATUS "jbpf: ${JBPF_DIR} @ ${_jbpf_rev} (${_jbpf_clock})") +unset(_jbpf_rev) +unset(_jbpf_pin) +unset(_jbpf_mono) +unset(_jbpf_clock) + # Override add_test and set_tests_properties so jbpf subdir doesn't register tests macro(add_test) endmacro() macro(set_tests_properties) endmacro() -add_subdirectory(jbpf) +# JBPF_DIR is outside this source tree, so add_subdirectory needs an explicit +# binary directory. jbpf writes its libraries and copied headers under +# ${CMAKE_SOURCE_DIR}/out (this project's out/). Two things still land in ocudu's +# tree, as they do in ocudu's own build: libck, which jbpf builds in-source in +# 3p/ck, and src/common/jbpf_version.h, which it regenerates on every build (build +# parameters as strings only). Do not build ocudu and the controller at the same +# time from one checkout. +add_subdirectory("${JBPF_DIR}" "${CMAKE_BINARY_DIR}/jbpf") # Restore original cmake commands unset(add_test) @@ -100,10 +178,7 @@ unset(set_tests_properties) # # It needs ocudu's janus headers, which define the hook context structs — the # verifier derives its ctx descriptors from them with offsetof() so the -# contract has a single source of truth. Point OCUDU_DIR elsewhere if your -# checkout is not a sibling of this one. -set(OCUDU_DIR "${CMAKE_SOURCE_DIR}/../ocudu-e3" - CACHE PATH "Path to the ocudu-e3 checkout (hook context contract)") +# contract has a single source of truth. They come from OCUDU_DIR (above). set(OCUDU_JANUS_INC "${OCUDU_DIR}/include/ocudu/janus") add_subdirectory(codelets/verifier) @@ -208,12 +283,12 @@ target_include_directories(e3_controller PRIVATE ${CMAKE_BINARY_DIR}/generated # libe3_stamp.h ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/src - ${CMAKE_SOURCE_DIR}/jbpf/src/io - ${CMAKE_SOURCE_DIR}/jbpf/src/mem_mgmt - ${CMAKE_SOURCE_DIR}/jbpf/src/logger - ${CMAKE_SOURCE_DIR}/jbpf/src/common + ${JBPF_DIR}/src/io + ${JBPF_DIR}/src/mem_mgmt + ${JBPF_DIR}/src/logger + ${JBPF_DIR}/src/common ${OUTPUT_DIR}/inc - ${CMAKE_SOURCE_DIR}/jbpf/src/lcm + ${JBPF_DIR}/src/lcm ) # Link against jbpf_io (transitively brings mem_mgmt, logger, ck, mimalloc) diff --git a/README.md b/README.md index bcbfd92..49a8332 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,14 @@ thread pinning — lives in a single YAML configuration file; see [Usage](#usage ## Build -Everything goes through the top-level [`build.sh`](build.sh). Two modes: +Everything goes through the top-level [`build.sh`](build.sh). The controller is +built against the gNB's own jbpf, so clone [ocudu-e3](https://github.com/wineslab/ocudu-e3) +next to it first (see [jbpf comes from ocudu](#jbpf-comes-from-ocudu)). Two modes: ```bash +git clone --branch e3 https://github.com/wineslab/ocudu-e3.git +git -C ocudu-e3 submodule update --init --recursive external/jbpf + git clone --recurse-submodules git@github.com:wineslab/E3Controller.git cd E3Controller @@ -56,16 +61,16 @@ packages by hand and re-run `./build.sh` without `--install-deps`. Whether or not `--install-deps` was used, `build.sh` then runs: -1. `git submodule update --init --recursive` (fetches libe3 and jbpf at their - pinned commits). -2. `jbpf/init_and_patch_submodules.sh` for jbpf's third-party dependencies, then - `apply_jbpf_patches.sh` for the patches to jbpf's own core. +1. `git submodule update --init --recursive` (fetches libe3 at its pinned tag). +2. Checks that ocudu's `external/jbpf` and its third-party submodules are + populated, then runs ocudu's `apply_jbpf_patches.sh` over them. That script + is idempotent, so on a tree the gNB build already patched it does nothing. 3. Configure + build libe3 with both encoders (`-DLIBE3_ENABLE_ASN1=ON -DLIBE3_ENABLE_JSON=ON -DLIBE3_BUILD_EXAMPLES=OFF -DLIBE3_BUILD_TESTS=OFF`), then `sudo cmake --install libe3/build` to `/usr/local`. -4. Configure + build the E3Controller and the recording-cost bench; jbpf is - compiled in-tree via `add_subdirectory`. +4. Configure + build the E3Controller and the recording-cost bench; ocudu's + jbpf is compiled into this build tree via `add_subdirectory`. `e3.encoding` in the config is therefore a pure runtime choice, because libe3 is built with both encoders. @@ -90,6 +95,12 @@ behaviour of an installed public header. ### Overrides & re-runs +- `OCUDU_DIR=/path/to/ocudu ./build.sh` — the ocudu checkout to take jbpf and + the hook contract from (defaults to `../ocudu-e3`; a relative path is taken + from this repository's root). +- `LIBE3_PREFIX=$HOME/.local ./build.sh` — install libe3 there instead of + `/usr/local`, without `sudo`, and build the controller against it. For hosts + where you are not root. - `JOBS=N ./build.sh` — parallelism (defaults to `nproc`). - `E3C_CMAKE_ARGS='-DUSE_NATIVE=OFF' ./build.sh` — extra configure flags for the controller. CI passes exactly this: `-march=native` is right on a @@ -100,6 +111,30 @@ behaviour of an installed public header. put `asn1c` under `/opt/asn1c/`, the script re-adds `/opt/asn1c/bin` to `PATH` automatically before invoking `cmake`. +### jbpf comes from ocudu + +The controller is the jbpf IPC primary and the gNB the secondary. Both map one +shared region and exchange one channel and message format, so they must be built +from the same jbpf. This repository therefore has no jbpf of its own: it builds +against `${OCUDU_DIR}/external/jbpf`, the tree the gNB is built from, with the +options ocudu uses (`JBPF_STATIC=ON`, `JBPF_EXPERIMENTAL_FEATURES=ON`). The +patches to that tree belong to ocudu too (`jbpf_patches/`, applied by its +`apply_jbpf_patches.sh`). + +Configure reports which jbpf it found, and warns when: + +- that jbpf is at a different commit than the one ocudu pins, or +- it lacks `jbpf_monotonic_time.patch`. The controller does not need the patch + itself, but a gNB built from the same tree would stamp `codelet_ts_ns` with + `CLOCK_REALTIME` while `gnb_ts_ns` is `CLOCK_MONOTONIC`. + +jbpf builds `libck` inside its own source tree (`3p/ck`), as it does in ocudu's +build, so do not build ocudu and the controller at the same time from one +checkout. Run one after the other. + +The codelet Makefiles follow the same rule: `JBPF_DIR` defaults to +`$(OCUDU_DIR)/external/jbpf`. + ### ASN.1 Code Generation The build automatically: @@ -308,7 +343,7 @@ nothing yet copied, and `codelet_ts_ns` just before the codelet submits. So the controller replays them rather than the RAN keeping a ring of its own. That works because the whole chain reads `CLOCK_MONOTONIC` — the gNB hook, -`jbpf_time_get_ns()` (see `jbpf_patches/jbpf_monotonic_time.patch`) and the +`jbpf_time_get_ns()` (see ocudu's `jbpf_patches/jbpf_monotonic_time.patch`) and the dispatcher poll — which is latrec's own clock. No domain conversion, no offset arithmetic, no rate skew to absorb. **All of those sites have to agree**; if one drifts back to `CLOCK_REALTIME` the stage intervals silently mix epochs. diff --git a/apply_jbpf_patches.sh b/apply_jbpf_patches.sh deleted file mode 100755 index bdbe067..0000000 --- a/apply_jbpf_patches.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# -# Apply this repo's patches against the jbpf submodule. -# -# Run after `git submodule update --init --recursive` and before configuring. -# build.sh calls it as step 2; it is also safe to run by hand. -# -# Idempotent: an already-patched tree is detected and skipped. -set -euo pipefail - -REPO_DIR="$(cd "$(dirname "$0")" && pwd)" -JBPF_DIR="${REPO_DIR}/jbpf" -OURS_DIR="${REPO_DIR}/jbpf_patches" - -if [ ! -f "${JBPF_DIR}/CMakeLists.txt" ]; then - echo "ERROR: jbpf submodule at ${JBPF_DIR} is not populated." >&2 - echo " Run: git submodule update --init --recursive" >&2 - exit 1 -fi - -# run_patch -# -# One call site for both back-ends so the two never drift in what they -# consider "applies". Returns the tool's exit status; prints nothing. -run_patch() { - local mode="$1" tree="$2" file="$3" dir="$4" kind="$5" - local -a args=() - if [ "${dir}" = reverse ]; then args+=(--reverse); fi - - # ${args[@]+...}: a forward+real call leaves args empty, and bash 3.2 (still - # the /bin/bash on macOS) treats "${args[@]}" as unbound under `set -u`. - if [ "${mode}" = git ]; then - if [ "${kind}" = dry ]; then args+=(--check); fi - git -C "${tree}" apply ${args[@]+"${args[@]}"} "${file}" >/dev/null 2>&1 - else - if [ "${kind}" = dry ]; then args+=(--dry-run); fi - ( cd "${tree}" && patch -p1 --silent --force --no-backup-if-mismatch \ - ${args[@]+"${args[@]}"} <"${file}" ) >/dev/null 2>&1 - fi -} - -apply_patch() { - local patch="$1" - local patch_file="${OURS_DIR}/${patch}" - - if [ ! -f "${patch_file}" ]; then - echo "ERROR: ${patch} is missing from ${OURS_DIR}." >&2 - exit 1 - fi - - # git apply where the tree is a real checkout, patch(1) where it is not. - # The fallback is not hypothetical: deployment rsyncs this tree into the pod - # without .git, and there `git apply` fails at repository DISCOVERY -- - # indistinguishable, from the exit status alone, from a conflicting patch. - local mode=git - git -C "${JBPF_DIR}" rev-parse --git-dir >/dev/null 2>&1 || mode=patch - - if run_patch "${mode}" "${JBPF_DIR}" "${patch_file}" reverse dry; then - echo "==> jbpf: ${patch} already applied, skipping" - elif run_patch "${mode}" "${JBPF_DIR}" "${patch_file}" forward dry; then - echo "==> jbpf: applying ${patch}" - run_patch "${mode}" "${JBPF_DIR}" "${patch_file}" forward real - else - echo "ERROR: ${patch} does not apply cleanly to jbpf (nor is it already" >&2 - echo " applied). The tree may have local modifications; inspect it," >&2 - echo " or re-init with:" >&2 - echo " git submodule update --init --recursive --force jbpf" >&2 - exit 1 - fi -} - -# Upstream jbpf_time_get_ns() stamps CLOCK_REALTIME, and that is the clock the -# slot codelet writes into codelet_ts_ns. The gNB, this controller and the dApp -# all read CLOCK_MONOTONIC, so leaving it unpatched puts codelet_ts_ns in a -# different epoch from gnb_ts_ns: gnb_to_codelet_us then comes out silently -# wrong rather than failing. -apply_patch jbpf_monotonic_time.patch - -echo "==> All E3Controller jbpf patches applied." diff --git a/build.sh b/build.sh index 1b1af8e..9869f69 100755 --- a/build.sh +++ b/build.sh @@ -2,8 +2,8 @@ # # One-shot build for the E3Controller. # -# Build order (libe3 depends on nothing in-tree; the controller depends on a -# system-installed libe3 + an in-tree jbpf): +# Build order (libe3 depends on nothing in-tree; the controller depends on an +# installed libe3 + the gNB's jbpf, from the ocudu checkout next to this one): # # 0. (optional, --install-deps) install system packages. This step delegates # to libe3's own `libe3/build_libe3 -I`, which does an apt install of the @@ -11,11 +11,12 @@ # the mouse07410 fork into /opt/asn1c so the file list libe3's E3AP # grammar expects is produced. We also apt-install a small set of extras # the E3Controller itself needs (python + pip for jbpf's build). -# 1. fetch submodules (libe3 @ pinned tag, jbpf) -# 2. init + patch jbpf's own 3p submodules, then apply our patch to jbpf core -# 3. configure libe3, stage asn1c's BOOLEAN.* skeletons (toolchain workaround), -# then build + INSTALL to /usr/local (both encoders -> runtime --encoding) -# 4. build the E3Controller (jbpf is built in-tree via add_subdirectory) +# 1. fetch the libe3 submodule (pinned tag) +# 2. check that ocudu's external/jbpf is populated, then run ocudu's +# apply_jbpf_patches.sh over it (idempotent: a no-op on a patched tree) +# 3. configure libe3, then build + INSTALL it (to /usr/local, or LIBE3_PREFIX) +# with both encoders, so the encoding is a runtime choice +# 4. build the E3Controller (ocudu's jbpf is built via add_subdirectory) # # Usage: ./build.sh [--install-deps] [--latrec] # --install-deps Install all system packages before building. Debian/Ubuntu @@ -30,8 +31,17 @@ # build. Set LATREC_DEFAULT_DIR= alongside it to move # the compiled-in default ring directory off /tmp/latrec. # -# Override parallelism with JOBS= ./build.sh. -# Extra configure flags for the controller: E3C_CMAKE_ARGS='-DUSE_NATIVE=OFF'. +# Environment: +# OCUDU_DIR= The ocudu checkout the gNB is built from (default +# ../ocudu-e3, relative to this repository). Its +# external/jbpf is the jbpf this build uses, and its +# include/ocudu/janus the hook contract. +# LIBE3_PREFIX= Install libe3 there instead of /usr/local, without sudo, +# and build the controller against it. For hosts where +# you are not root. +# JOBS= Parallelism (default: nproc). +# E3C_CMAKE_ARGS=... Extra configure flags for the controller, e.g. +# '-DUSE_NATIVE=OFF'. set -euo pipefail cd "$(dirname "$0")" @@ -41,7 +51,7 @@ for arg in "$@"; do case "$arg" in --install-deps|-d) INSTALL_DEPS=1 ;; --latrec) ENABLE_LATREC=1 ;; - -h|--help) sed -n '2,34p' "$0"; exit 0 ;; + -h|--help) sed -n '2,44p' "$0"; exit 0 ;; *) echo "ERROR: unknown argument '$arg'" >&2 echo "Usage: $0 [--install-deps] [--latrec]" >&2 exit 2 ;; @@ -134,23 +144,55 @@ fi # --------------------------------------------------------------------------- # Step 1: submodules # --------------------------------------------------------------------------- -echo "==> [1/4] Fetching submodules (libe3, jbpf)" +echo "==> [1/4] Fetching submodules (libe3)" git submodule update --init --recursive # --------------------------------------------------------------------------- -# Step 2: jbpf's own 3p submodules, then jbpf core +# Step 2: the gNB's jbpf, from the ocudu checkout # --------------------------------------------------------------------------- -echo "==> [2/4] Initialising + patching jbpf 3p submodules" -( cd jbpf && bash ./init_and_patch_submodules.sh ) +# The controller (jbpf IPC primary) and the gNB (secondary) must be built from +# the same jbpf, so this build uses ocudu's external/jbpf rather than a copy of +# its own. ocudu owns that tree and its patches; we only check that it is there +# and run ocudu's own, idempotent, patch script over it. +# A relative OCUDU_DIR is taken from this repository's root (we cd'd there). +OCUDU_DIR_IN="${OCUDU_DIR:-../ocudu-e3}" +if ! OCUDU_DIR="$(cd "${OCUDU_DIR_IN}" 2>/dev/null && pwd)"; then + echo "ERROR: no ocudu checkout at OCUDU_DIR='${OCUDU_DIR_IN}'." >&2 + echo " Clone it next to this repository, or point OCUDU_DIR at it:" >&2 + echo " git clone --branch e3 https://github.com/wineslab/ocudu-e3.git ../ocudu-e3" >&2 + echo " OCUDU_DIR=/path/to/ocudu ./build.sh" >&2 + exit 1 +fi +JBPF_DIR="${OCUDU_DIR}/external/jbpf" +echo "==> [2/4] Preparing the gNB's jbpf (${JBPF_DIR})" -# Then our own patches against jbpf core, which that script knows nothing -# about (it is upstream jbpf's, so edits to it are lost on re-clone). -bash ./apply_jbpf_patches.sh +# Same sentinels as CMakeLists.txt: one file each 3p module actually ships. +for sentinel in CMakeLists.txt 3p/ubpf/CMakeLists.txt 3p/ebpf-verifier/CMakeLists.txt \ + 3p/mimalloc/CMakeLists.txt 3p/ck/configure; do + if [ ! -f "${JBPF_DIR}/${sentinel}" ]; then + echo "ERROR: ${JBPF_DIR}/${sentinel} is missing: ocudu's jbpf submodule is" >&2 + echo " not populated. On the ocudu side, run:" >&2 + echo " git -C ${OCUDU_DIR} submodule update --init --recursive external/jbpf" >&2 + exit 1 + fi +done + +if [ ! -f "${OCUDU_DIR}/apply_jbpf_patches.sh" ]; then + echo "ERROR: ${OCUDU_DIR}/apply_jbpf_patches.sh not found; this ocudu checkout" >&2 + echo " predates the jbpf integration. Use the e3 branch." >&2 + exit 1 +fi +bash "${OCUDU_DIR}/apply_jbpf_patches.sh" # --------------------------------------------------------------------------- # Step 3: libe3 # --------------------------------------------------------------------------- -echo "==> [3/4] Building + installing libe3 (ASN.1 + JSON) to /usr/local" +LIBE3_PREFIX="${LIBE3_PREFIX:-}" +if [ -n "${LIBE3_PREFIX}" ]; then + mkdir -p "${LIBE3_PREFIX}" + LIBE3_PREFIX="$(cd "${LIBE3_PREFIX}" && pwd)" +fi +echo "==> [3/4] Building + installing libe3 (ASN.1 + JSON) to ${LIBE3_PREFIX:-/usr/local}" # JSON encoding needs nlohmann_json >= 3.11 on the system (libe3's floor); install # it (header-only) or let libe3's FetchContent fetch it when the host has internet. # @@ -175,6 +217,9 @@ LIBE3_CMAKE_ARGS=( -DLIBE3_BUILD_EXAMPLES=OFF -DLIBE3_BUILD_TESTS=OFF ) +if [ -n "${LIBE3_PREFIX}" ]; then + LIBE3_CMAKE_ARGS+=( -DCMAKE_INSTALL_PREFIX="${LIBE3_PREFIX}" ) +fi if [ "$ENABLE_LATREC" -eq 1 ]; then echo " latrec: ON (stage recorder compiled in)" LIBE3_CMAKE_ARGS+=( -DLIBE3_ENABLE_LATREC=ON ) @@ -198,7 +243,11 @@ cmake -S libe3 -B libe3/build "${LIBE3_CMAKE_ARGS[@]}" # reference skeletons on disk. cmake --build libe3/build -j"${JOBS}" -$SUDO cmake --install libe3/build +if [ -n "${LIBE3_PREFIX}" ]; then + cmake --install libe3/build +else + $SUDO cmake --install libe3/build +fi # --------------------------------------------------------------------------- # Step 4: E3Controller @@ -208,8 +257,16 @@ echo "==> [4/4] Building E3Controller" # CI sets -DUSE_NATIVE=OFF: -march=native is right on a deployment host but wrong # for a measurement run on whatever CPU a shared runner happens to be, since it # makes numbers incomparable between runs. +E3C_PREFIX_ARGS=() +if [ -n "${LIBE3_PREFIX}" ]; then + # find_package(libe3) and the ASN.1 runtime headers (src/e3sm/asn) both have to + # look in the prefix, or they silently pick up an older libe3 in /usr/local. + E3C_PREFIX_ARGS+=( -DCMAKE_PREFIX_PATH="${LIBE3_PREFIX}" + -DLIBE3_ASN1_INCLUDE_DIR="${LIBE3_PREFIX}/include/libe3/asn1" ) +fi # shellcheck disable=SC2086 -cmake -S . -B build -DINITIALIZE_SUBMODULES=OFF ${E3C_CMAKE_ARGS:-} +cmake -S . -B build -DOCUDU_DIR="${OCUDU_DIR}" \ + ${E3C_PREFIX_ARGS[@]+"${E3C_PREFIX_ARGS[@]}"} ${E3C_CMAKE_ARGS:-} cmake --build build -j"${JOBS}" --target e3_controller bench_stage_recording echo "==> Done: $(pwd)/out/bin/e3_controller" diff --git a/codelets/Makefile.defs b/codelets/Makefile.defs index 0528520..27ebf8e 100644 --- a/codelets/Makefile.defs +++ b/codelets/Makefile.defs @@ -18,11 +18,7 @@ CODELETS_DIR := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST))))) E3_ROOT := $(patsubst %/,%,$(dir $(CODELETS_DIR))) -# ---- jbpf (the E3Controller submodule; built by build.sh into $(E3_ROOT)/out) ---- -JBPF_DIR ?= $(E3_ROOT)/jbpf -JBPF_OUT_DIR ?= $(E3_ROOT)/out - -# ---- ocudu: source of truth for the hook context contract ---- +# ---- ocudu: source of truth for the hook context contract, and for jbpf ---- # The ctx structs are defined by the hook PRODUCER, so ocudu owns them. We put # its headers on the include path rather than keeping a copy here. Three # byte-identical copies of jbpf_srsran_contexts.h used to float around the @@ -32,6 +28,12 @@ OCUDU_DIR ?= $(E3_ROOT)/../ocudu-e3 OCUDU_JANUS_INC ?= $(OCUDU_DIR)/include/ocudu/janus OCUDU_SPECS_INC ?= $(OCUDU_DIR)/ocudu_janus/verifier/specs +# ---- jbpf: ocudu's external/jbpf, the tree the gNB that loads these codelets is +# built from (see the top-level CMakeLists.txt). Its generated headers land in +# $(E3_ROOT)/out, where the controller's cmake build puts them. ---- +JBPF_DIR ?= $(OCUDU_DIR)/external/jbpf +JBPF_OUT_DIR ?= $(E3_ROOT)/out + # Local import target for environments without an ocudu checkout (e.g. a build # container). `make import-contract` populates it; `make check-contract` fails # on drift. OCUDU_JANUS_INC comes first on the include path so the live tree @@ -45,9 +47,11 @@ E3_VERIFIER ?= $(JBPF_OUT_DIR)/bin/e3_verifier_cli # with `$(MAKE) -C `, so a relative override like # `make OCUDU_DIR=../ocudu-e3` would otherwise be re-resolved against each # subdirectory's CWD and silently fail to find the contract. +# +# OCUDU_DIR first: JBPF_DIR defaults to a path under it. +override OCUDU_DIR := $(abspath $(OCUDU_DIR)) override JBPF_DIR := $(abspath $(JBPF_DIR)) override JBPF_OUT_DIR := $(abspath $(JBPF_OUT_DIR)) -override OCUDU_DIR := $(abspath $(OCUDU_DIR)) override OCUDU_JANUS_INC := $(abspath $(OCUDU_JANUS_INC)) override OCUDU_SPECS_INC := $(abspath $(OCUDU_SPECS_INC)) override E3_VERIFIER := $(abspath $(E3_VERIFIER)) @@ -134,10 +138,10 @@ check-env: echo "ERROR: jbpf sources not found at JBPF_DIR=$(JBPF_DIR)" >&2; \ echo " Expected $(JBPF_DIR)/src/common/jbpf_defs.h" >&2; \ if [ -d "$(JBPF_DIR)" ] && [ -z "$$(ls -A $(JBPF_DIR) 2>/dev/null)" ]; then \ - echo " That directory is EMPTY — the submodule is not checked out:" >&2; \ - echo " git -C $(E3_ROOT) submodule update --init --recursive" >&2; \ + echo " That directory is EMPTY — ocudu's jbpf submodule is not checked out:" >&2; \ + echo " git -C $(OCUDU_DIR) submodule update --init --recursive external/jbpf" >&2; \ else \ - echo " Override it: make JBPF_DIR=/path/to/jbpf" >&2; \ + echo " Point OCUDU_DIR at the ocudu checkout: make OCUDU_DIR=/path/to/ocudu" >&2; \ fi; \ echo " Run 'make show-config' to see every resolved path." >&2; \ exit 1; \ diff --git a/codelets/uplink_slot_samples/README.md b/codelets/uplink_slot_samples/README.md index bdb8bbd..a1eb597 100644 --- a/codelets/uplink_slot_samples/README.md +++ b/codelets/uplink_slot_samples/README.md @@ -85,7 +85,7 @@ invariant rather than a local choice: | site | file | |---|---| | `gnb_ts_ns` | `lib/phy/upper/upper_phy_rx_symbol_handler_impl.cpp` | -| `codelet_ts_ns`, via `jbpf_time_get_ns()` | `jbpf/src/core/jbpf_helper_impl.c` — **local patch**, see `jbpf_monotonic_time.patch` | +| `codelet_ts_ns`, via `jbpf_time_get_ns()` | ocudu `external/jbpf/src/core/jbpf_helper_impl.c` — **local patch**, see ocudu's `jbpf_patches/jbpf_monotonic_time.patch` | | `dispatch_ts_ns` | `E3Controller/src/e3sm/slot_iq_pipeline.cpp` | | handler entry | `E3Controller/src/e3sm/l1_kpm/e3sm_layer_1.cpp` | | the dApp's arrival age | `adaptive_cpu/subcarrier_power_app.cpp` (`now_mono_ns`) | diff --git a/codelets/verifier/CMakeLists.txt b/codelets/verifier/CMakeLists.txt index e75071f..bc3c5ea 100644 --- a/codelets/verifier/CMakeLists.txt +++ b/codelets/verifier/CMakeLists.txt @@ -1,7 +1,8 @@ # e3_verifier_cli — offline PREVAIL verifier for E3Controller's codelets. # -# Replaces the SDK image's srsran_verifier_cli. Built from the jbpf submodule's -# verifier library, plus the E3 program-type and helper registrations. +# Replaces the SDK image's srsran_verifier_cli. Built from the verifier library +# of ocudu's jbpf (JBPF_DIR, set in the top-level CMakeLists.txt), plus the E3 +# program-type and helper registrations. # # Needs ocudu's janus headers for the ctx struct definitions: the context # descriptors are derived with offsetof() from the RAN's own header rather than @@ -20,7 +21,7 @@ add_executable(e3_verifier_cli e3_verifier_cli.cpp) target_link_libraries(e3_verifier_cli PRIVATE jbpf::verifier_lib) -set(_e3v_jbpf "${CMAKE_CURRENT_SOURCE_DIR}/../../jbpf") +set(_e3v_jbpf "${JBPF_DIR}") target_include_directories(e3_verifier_cli PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include # jbpf_e3_ids.h ${OCUDU_JANUS_INC} # jbpf_srsran_contexts.h (contract) diff --git a/jbpf b/jbpf deleted file mode 160000 index b15dd03..0000000 --- a/jbpf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b15dd0306b8be9ce1217f9de84411c36699de2d3 diff --git a/jbpf_patches/jbpf_monotonic_time.patch b/jbpf_patches/jbpf_monotonic_time.patch deleted file mode 100644 index 7d80260..0000000 --- a/jbpf_patches/jbpf_monotonic_time.patch +++ /dev/null @@ -1,25 +0,0 @@ ---- a/src/core/jbpf_helper_impl.c -+++ b/src/core/jbpf_helper_impl.c -@@ -328,7 +328,21 @@ - { - struct timespec curr_time = {0, 0}; - uint64_t curr_time_ns = 0; -- clock_gettime(CLOCK_REALTIME, &curr_time); -+ /* CLOCK_MONOTONIC, not CLOCK_REALTIME (LOCAL PATCH -- see -+ * jbpf_monotonic_time.patch in the ocudu_project root). -+ * -+ * This helper stamps the E3 codelet's codelet_ts_ns, which is subtracted -+ * from the gNB's gnb_ts_ns to produce gnb_to_codelet_us. The gNB, the -+ * E3Controller and the dApp all read CLOCK_MONOTONIC now, and it is also the -+ * clock latrec records, so every stage of the path shares one epoch and the -+ * stage CSVs join the latrec rings with no offset arithmetic. -+ * -+ * Monotonic additionally cannot be stepped by NTP, which REALTIME can -- -+ * a step mid-run silently corrupts every stage derived from these stamps. -+ * -+ * Caveat: monotonic shares an origin only within one boot, so stamps are -+ * comparable across processes on ONE host, not across hosts. */ -+ clock_gettime(CLOCK_MONOTONIC, &curr_time); - curr_time_ns = curr_time.tv_nsec + curr_time.tv_sec * 1.0e9; - return curr_time_ns; - }