From d61cf98782b842463707a40ad4d412fdcccdee19 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 14:35:58 -0700 Subject: [PATCH 01/13] Refactor build_bundle.sh into testable functions with fail-fast No behavior change for single-mode use: same env vars, messages, and artifacts. Adds set -euo pipefail so a failed wget/gtfstidy/zip/java now aborts the build instead of silently continuing, and BUNDLE_DIR (default /bundle) so tests don't need to touch /bundle. Lays the function structure (validate_mode_env, bundle_mode, fetch_url, run_single_mode, main) that later multi-input mode tasks build on. --- bin/build_bundle_test.sh | 118 +++++++++++++++++++++++++++++++++ bundler/build_bundle.sh | 139 ++++++++++++++++++++++++++------------- oba/build_bundle.sh | 139 ++++++++++++++++++++++++++------------- 3 files changed, 308 insertions(+), 88 deletions(-) create mode 100755 bin/build_bundle_test.sh diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh new file mode 100755 index 0000000..8fc0254 --- /dev/null +++ b/bin/build_bundle_test.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Unit tests for oba/build_bundle.sh. Sources the script (main guard prevents +# execution) and exercises functions with stub binaries on PATH. +# Style follows bin/e2e_api_key_test.sh (pass/fail counters). + +set -u + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +SUT="$REPO_ROOT/oba/build_bundle.sh" + +passed=0 +failed=0 + +pass() { echo "PASS: $1"; passed=$((passed + 1)); } +fail() { echo "FAIL: $1"; failed=$((failed + 1)); } + +# The harness deliberately runs WITHOUT errexit: failing invocations of the +# script under test are expected and asserted on. + +# Run the script as a subprocess with a controlled env. +# usage: run_script [VAR=value ...] +# stdout+stderr -> $RUN_OUTPUT, exit code -> $RUN_STATUS +run_script() { + local stub_dir="$1"; shift + local tmp_out + tmp_out="$(mktemp)" + local path_prefix="" + [ -n "$stub_dir" ] && path_prefix="$stub_dir:" + env -i PATH="${path_prefix}/usr/bin:/bin:/usr/sbin:/sbin" HOME="$HOME" "$@" bash "$SUT" >"$tmp_out" 2>&1 + RUN_STATUS=$? + RUN_OUTPUT="$(cat "$tmp_out")" + rm -f "$tmp_out" +} + +assert_contains() { + local haystack="$1" needle="$2" label="$3" + if printf '%s' "$haystack" | grep -qF -- "$needle"; then + pass "$label" + else + fail "$label — expected to find: $needle" + printf 'GOT:\n%s\n' "$haystack" + fi +} + +echo "=== build_bundle.sh unit tests ===" + +# --- Mode validation: existing single-mode errors preserved ----------------- + +run_script "" GTFS_URL=http://x/gtfs.zip GTFS_ZIP_FILENAME=local.zip +[ "$RUN_STATUS" -ne 0 ] && pass "both GTFS_URL and GTFS_ZIP_FILENAME exits nonzero" \ + || fail "both GTFS_URL and GTFS_ZIP_FILENAME exits nonzero" +assert_contains "$RUN_OUTPUT" "Both GTFS_URL and GTFS_ZIP_FILENAME are set" "both-set error message preserved" + +run_script "" +[ "$RUN_STATUS" -ne 0 ] && pass "no mode env exits nonzero" || fail "no mode env exits nonzero" +assert_contains "$RUN_OUTPUT" "Neither GTFS_URL nor GTFS_ZIP_FILENAME is set" "neither-set error message preserved" + +# --- Single mode happy path with stubbed binaries --------------------------- + +STUBS="$(mktemp -d)" +WORK="$(mktemp -d)" + +cat > "$STUBS/wget" <<'EOF' +#!/bin/bash +# stub wget: records argv; creates the -O target +out="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else shift; fi +done +echo "wget $out" >> "$STUB_LOG" +echo "fake-zip" > "$out" +EOF + +cat > "$STUBS/gtfstidy" <<'EOF' +#!/bin/bash +echo "gtfstidy $*" >> "$STUB_LOG" +EOF + +cat > "$STUBS/java" <<'EOF' +#!/bin/bash +echo "java $*" >> "$STUB_LOG" +echo "cwd $PWD" >> "$STUB_LOG" +EOF + +chmod +x "$STUBS/wget" "$STUBS/gtfstidy" "$STUBS/java" + +STUB_LOG="$WORK/stub.log" +: > "$STUB_LOG" + +run_script "$STUBS" STUB_LOG="$STUB_LOG" BUNDLE_DIR="$WORK" \ + GTFS_URL=http://example.test/gtfs.zip TDF_BUILDER_JAR=/fake/builder.jar OBA_VERSION=test +[ "$RUN_STATUS" -eq 0 ] && pass "single mode (stubbed) exits 0" || fail "single mode (stubbed) exits 0: $RUN_OUTPUT" +assert_contains "$(cat "$STUB_LOG")" "wget $WORK/gtfs_pristine.zip" "single mode downloads to gtfs_pristine.zip" +assert_contains "$(cat "$STUB_LOG")" "gtfstidy" "single mode still runs gtfstidy" +assert_contains "$(cat "$STUB_LOG")" "java -Xss4m -Xmx3g -jar /fake/builder.jar ./gtfs_pristine.zip ." "single mode builder argv unchanged" +assert_contains "$(cat "$STUB_LOG")" "cwd $WORK" "builder runs from bundle dir" + +# --- Failed download now aborts (set -e hardening) --------------------------- + +cat > "$STUBS/wget" <<'EOF' +#!/bin/bash +exit 8 +EOF +chmod +x "$STUBS/wget" + +run_script "$STUBS" STUB_LOG="$STUB_LOG" BUNDLE_DIR="$WORK" GTFS_URL=http://example.test/gtfs.zip +[ "$RUN_STATUS" -ne 0 ] && pass "failed GTFS download aborts the build" || fail "failed GTFS download aborts the build" +assert_contains "$RUN_OUTPUT" "ERROR:" "failed download prints an ERROR: line" + +rm -rf "$STUBS" "$WORK" + +echo "" +echo "==============================" +echo "Results: $passed passed, $failed failed" +echo "==============================" +[ "$failed" -eq 0 ] diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index 0cc77dc..80df809 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -16,17 +16,14 @@ # limitations under the License. # -# Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both -if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." - exit 1 -fi - -if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." - exit 1 -fi +set -euo pipefail +# Normalize env so `set -u` can't trip on optional vars. +GTFS_URL=${GTFS_URL:-} +GTFS_ZIP_FILENAME=${GTFS_ZIP_FILENAME:-} +BUNDLE_INPUTS_URL=${BUNDLE_INPUTS_URL:-} +STOP_CONSOLIDATION_URL=${STOP_CONSOLIDATION_URL:-} +OBA_VERSION=${OBA_VERSION:-} TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation-builder-withAllDependencies.jar} # Run gtfstidy (https://github.com/patrickbr/gtfstidy) with the following options enabled by default: @@ -41,43 +38,97 @@ TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation- # -D: drop erroneous entries from feed GTFS_TIDY_ARGS=${GTFS_TIDY_ARGS:-OscRCSmeD} -# Set default filename if using GTFS_URL -if [ -n "$GTFS_URL" ]; then - GTFS_ZIP_FILENAME="gtfs_pristine.zip" -fi +# Overridable for tests; production always uses /bundle. +BUNDLE_DIR=${BUNDLE_DIR:-/bundle} -echo "OBA Bundle Builder Starting" -if [ -n "$GTFS_URL" ]; then - echo "GTFS_URL: $GTFS_URL" -else - echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" -fi -echo "OBA Version: $OBA_VERSION" -echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" -echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" - -cd /bundle - -# Download GTFS file if URL is provided, otherwise use local file -if [ -n "$GTFS_URL" ]; then - wget -O ${GTFS_ZIP_FILENAME} ${GTFS_URL} -else - # Check if the local file exists - if [ ! -f "$GTFS_ZIP_FILENAME" ]; then - echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" +validate_mode_env() { + if [ -n "$BUNDLE_INPUTS_URL" ] && { [ -n "$GTFS_URL" ] || [ -n "$GTFS_ZIP_FILENAME" ]; }; then + echo "Error: BUNDLE_INPUTS_URL cannot be combined with GTFS_URL or GTFS_ZIP_FILENAME. Please provide only one mode." exit 1 fi -fi -gtfstidy -${GTFS_TIDY_ARGS} ${GTFS_ZIP_FILENAME} + if [ -n "$BUNDLE_INPUTS_URL" ]; then + return 0 + fi -if [[ -d "gtfs-out" ]]; then - cd gtfs-out - zip ../gtfs_tidied.zip * - cd .. - GTFS_ZIP_FILENAME="gtfs_tidied.zip" -fi + # Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both + if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." + exit 1 + fi + + if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." + exit 1 + fi +} + +bundle_mode() { + if [ -n "$BUNDLE_INPUTS_URL" ]; then + echo "multi" + else + echo "single" + fi +} -# The JAR must be executed from within the same directory -# as the bundle, or else some necessary files are not generated. -java -Xss4m -Xmx3g -jar $TDF_BUILDER_JAR ./${GTFS_ZIP_FILENAME} . +# fetch_url URL DEST LABEL — download with a one-line diagnostic on failure. +fetch_url() { + local url="$1" dest="$2" label="$3" + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download ${label} from ${url}" >&2 + exit 1 + fi +} + +run_single_mode() { + # Set default filename if using GTFS_URL + if [ -n "$GTFS_URL" ]; then + GTFS_ZIP_FILENAME="gtfs_pristine.zip" + fi + + echo "OBA Bundle Builder Starting" + if [ -n "$GTFS_URL" ]; then + echo "GTFS_URL: $GTFS_URL" + else + echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" + fi + echo "OBA Version: $OBA_VERSION" + echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + + cd "$BUNDLE_DIR" + + # Download GTFS file if URL is provided, otherwise use local file + if [ -n "$GTFS_URL" ]; then + fetch_url "$GTFS_URL" "$BUNDLE_DIR/$GTFS_ZIP_FILENAME" "GTFS feed" + else + # Check if the local file exists + if [ ! -f "$GTFS_ZIP_FILENAME" ]; then + echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" + exit 1 + fi + fi + + gtfstidy -"${GTFS_TIDY_ARGS}" "${GTFS_ZIP_FILENAME}" + + if [[ -d "gtfs-out" ]]; then + cd gtfs-out + zip ../gtfs_tidied.zip * + cd .. + GTFS_ZIP_FILENAME="gtfs_tidied.zip" + fi + + # The JAR must be executed from within the same directory + # as the bundle, or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" . +} + +main() { + validate_mode_env + run_single_mode +} + +# Main guard: allow tests to `source` this file without executing. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index 0cc77dc..80df809 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -16,17 +16,14 @@ # limitations under the License. # -# Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both -if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." - exit 1 -fi - -if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." - exit 1 -fi +set -euo pipefail +# Normalize env so `set -u` can't trip on optional vars. +GTFS_URL=${GTFS_URL:-} +GTFS_ZIP_FILENAME=${GTFS_ZIP_FILENAME:-} +BUNDLE_INPUTS_URL=${BUNDLE_INPUTS_URL:-} +STOP_CONSOLIDATION_URL=${STOP_CONSOLIDATION_URL:-} +OBA_VERSION=${OBA_VERSION:-} TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation-builder-withAllDependencies.jar} # Run gtfstidy (https://github.com/patrickbr/gtfstidy) with the following options enabled by default: @@ -41,43 +38,97 @@ TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation- # -D: drop erroneous entries from feed GTFS_TIDY_ARGS=${GTFS_TIDY_ARGS:-OscRCSmeD} -# Set default filename if using GTFS_URL -if [ -n "$GTFS_URL" ]; then - GTFS_ZIP_FILENAME="gtfs_pristine.zip" -fi +# Overridable for tests; production always uses /bundle. +BUNDLE_DIR=${BUNDLE_DIR:-/bundle} -echo "OBA Bundle Builder Starting" -if [ -n "$GTFS_URL" ]; then - echo "GTFS_URL: $GTFS_URL" -else - echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" -fi -echo "OBA Version: $OBA_VERSION" -echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" -echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" - -cd /bundle - -# Download GTFS file if URL is provided, otherwise use local file -if [ -n "$GTFS_URL" ]; then - wget -O ${GTFS_ZIP_FILENAME} ${GTFS_URL} -else - # Check if the local file exists - if [ ! -f "$GTFS_ZIP_FILENAME" ]; then - echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" +validate_mode_env() { + if [ -n "$BUNDLE_INPUTS_URL" ] && { [ -n "$GTFS_URL" ] || [ -n "$GTFS_ZIP_FILENAME" ]; }; then + echo "Error: BUNDLE_INPUTS_URL cannot be combined with GTFS_URL or GTFS_ZIP_FILENAME. Please provide only one mode." exit 1 fi -fi -gtfstidy -${GTFS_TIDY_ARGS} ${GTFS_ZIP_FILENAME} + if [ -n "$BUNDLE_INPUTS_URL" ]; then + return 0 + fi -if [[ -d "gtfs-out" ]]; then - cd gtfs-out - zip ../gtfs_tidied.zip * - cd .. - GTFS_ZIP_FILENAME="gtfs_tidied.zip" -fi + # Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both + if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." + exit 1 + fi + + if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." + exit 1 + fi +} + +bundle_mode() { + if [ -n "$BUNDLE_INPUTS_URL" ]; then + echo "multi" + else + echo "single" + fi +} -# The JAR must be executed from within the same directory -# as the bundle, or else some necessary files are not generated. -java -Xss4m -Xmx3g -jar $TDF_BUILDER_JAR ./${GTFS_ZIP_FILENAME} . +# fetch_url URL DEST LABEL — download with a one-line diagnostic on failure. +fetch_url() { + local url="$1" dest="$2" label="$3" + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download ${label} from ${url}" >&2 + exit 1 + fi +} + +run_single_mode() { + # Set default filename if using GTFS_URL + if [ -n "$GTFS_URL" ]; then + GTFS_ZIP_FILENAME="gtfs_pristine.zip" + fi + + echo "OBA Bundle Builder Starting" + if [ -n "$GTFS_URL" ]; then + echo "GTFS_URL: $GTFS_URL" + else + echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" + fi + echo "OBA Version: $OBA_VERSION" + echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + + cd "$BUNDLE_DIR" + + # Download GTFS file if URL is provided, otherwise use local file + if [ -n "$GTFS_URL" ]; then + fetch_url "$GTFS_URL" "$BUNDLE_DIR/$GTFS_ZIP_FILENAME" "GTFS feed" + else + # Check if the local file exists + if [ ! -f "$GTFS_ZIP_FILENAME" ]; then + echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" + exit 1 + fi + fi + + gtfstidy -"${GTFS_TIDY_ARGS}" "${GTFS_ZIP_FILENAME}" + + if [[ -d "gtfs-out" ]]; then + cd gtfs-out + zip ../gtfs_tidied.zip * + cd .. + GTFS_ZIP_FILENAME="gtfs_tidied.zip" + fi + + # The JAR must be executed from within the same directory + # as the bundle, or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" . +} + +main() { + validate_mode_env + run_single_mode +} + +# Main guard: allow tests to `source` this file without executing. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi From a819d250b0f040633356e4414094243086b07d29 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 14:44:22 -0700 Subject: [PATCH 02/13] Add BUNDLE_INPUTS_URL mode selection and env validation --- bin/build_bundle_test.sh | 21 +++++++++++++++++++-- bundler/build_bundle.sh | 18 +++++++++++++++++- oba/build_bundle.sh | 18 +++++++++++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh index 8fc0254..245d5a5 100755 --- a/bin/build_bundle_test.sh +++ b/bin/build_bundle_test.sh @@ -1,7 +1,8 @@ #!/bin/bash -# Unit tests for oba/build_bundle.sh. Sources the script (main guard prevents -# execution) and exercises functions with stub binaries on PATH. +# Unit tests for oba/build_bundle.sh. Runs the script as a subprocess +# (run_script; main DOES execute) and exercises functions with stub binaries +# on PATH. # Style follows bin/e2e_api_key_test.sh (pass/fail counters). set -u @@ -111,6 +112,22 @@ assert_contains "$RUN_OUTPUT" "ERROR:" "failed download prints an ERROR: line" rm -rf "$STUBS" "$WORK" +# --- Multi-input mode selection ---------------------------------------------- + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json GTFS_URL=http://x/gtfs.zip +[ "$RUN_STATUS" -ne 0 ] && pass "BUNDLE_INPUTS_URL + GTFS_URL exits nonzero" || fail "BUNDLE_INPUTS_URL + GTFS_URL exits nonzero" +assert_contains "$RUN_OUTPUT" "BUNDLE_INPUTS_URL cannot be combined" "combined-mode error message" + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json GTFS_ZIP_FILENAME=local.zip +[ "$RUN_STATUS" -ne 0 ] && pass "BUNDLE_INPUTS_URL + GTFS_ZIP_FILENAME exits nonzero" || fail "BUNDLE_INPUTS_URL + GTFS_ZIP_FILENAME exits nonzero" + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json STOP_CONSOLIDATION_URL=http://x/map.txt +[ "$RUN_STATUS" -ne 0 ] && pass "BUNDLE_INPUTS_URL + STOP_CONSOLIDATION_URL exits nonzero" || fail "BUNDLE_INPUTS_URL + STOP_CONSOLIDATION_URL exits nonzero" +assert_contains "$RUN_OUTPUT" "comes from the manifest" "consolidation-env-in-multi-mode error message" + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json +assert_contains "$RUN_OUTPUT" "Multi-input mode" "BUNDLE_INPUTS_URL alone selects multi mode" + echo "" echo "==============================" echo "Results: $passed passed, $failed failed" diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index 80df809..b91d2f7 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -47,6 +47,11 @@ validate_mode_env() { exit 1 fi + if [ -n "$BUNDLE_INPUTS_URL" ] && [ -n "$STOP_CONSOLIDATION_URL" ]; then + echo "Error: STOP_CONSOLIDATION_URL cannot be set in multi-input mode; the mapping comes from the manifest's stopConsolidationUrl." + exit 1 + fi + if [ -n "$BUNDLE_INPUTS_URL" ]; then return 0 fi @@ -123,9 +128,20 @@ run_single_mode() { java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" . } +run_multi_mode() { + echo "OBA Bundle Builder Starting" + echo "Multi-input mode: BUNDLE_INPUTS_URL: $BUNDLE_INPUTS_URL" + echo "ERROR: multi-input mode not yet implemented" >&2 + exit 1 +} + main() { validate_mode_env - run_single_mode + if [ "$(bundle_mode)" = "multi" ]; then + run_multi_mode + else + run_single_mode + fi } # Main guard: allow tests to `source` this file without executing. diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index 80df809..b91d2f7 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -47,6 +47,11 @@ validate_mode_env() { exit 1 fi + if [ -n "$BUNDLE_INPUTS_URL" ] && [ -n "$STOP_CONSOLIDATION_URL" ]; then + echo "Error: STOP_CONSOLIDATION_URL cannot be set in multi-input mode; the mapping comes from the manifest's stopConsolidationUrl." + exit 1 + fi + if [ -n "$BUNDLE_INPUTS_URL" ]; then return 0 fi @@ -123,9 +128,20 @@ run_single_mode() { java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" . } +run_multi_mode() { + echo "OBA Bundle Builder Starting" + echo "Multi-input mode: BUNDLE_INPUTS_URL: $BUNDLE_INPUTS_URL" + echo "ERROR: multi-input mode not yet implemented" >&2 + exit 1 +} + main() { validate_mode_env - run_single_mode + if [ "$(bundle_mode)" = "multi" ]; then + run_multi_mode + else + run_single_mode + fi } # Main guard: allow tests to `source` this file without executing. From 19d2140ab49d5a31f2f0528b684d0128cff34fe7 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 14:53:15 -0700 Subject: [PATCH 03/13] Download and validate bundle-inputs manifest, feeds, and mapping --- bin/build_bundle_test.sh | 102 ++++++++++++++++++ .../bundle-inputs-no-mapping.json | 7 ++ bin/testdata/build_bundle/bundle-inputs.json | 8 ++ bundler/build_bundle.sh | 59 ++++++++++ oba/build_bundle.sh | 59 ++++++++++ 5 files changed, 235 insertions(+) create mode 100644 bin/testdata/build_bundle/bundle-inputs-no-mapping.json create mode 100644 bin/testdata/build_bundle/bundle-inputs.json diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh index 245d5a5..0414bf5 100755 --- a/bin/build_bundle_test.sh +++ b/bin/build_bundle_test.sh @@ -128,6 +128,108 @@ assert_contains "$RUN_OUTPUT" "comes from the manifest" "consolidation-env-in-mu run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json assert_contains "$RUN_OUTPUT" "Multi-input mode" "BUNDLE_INPUTS_URL alone selects multi mode" +# --- download_bundle_inputs --------------------------------------------------- + +TESTDATA="$REPO_ROOT/bin/testdata/build_bundle" +STUBS2="$(mktemp -d)" +WORK2="$(mktemp -d)" +SERVE="$(mktemp -d)" + +cat > "$STUBS2/wget" <<'EOF' +#!/bin/bash +# stub wget: serves $SERVE/; 404s (exit 8) when absent +out="" url="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else url="$1"; shift; fi +done +src="$SERVE/$(basename "$url")" +if [ -f "$src" ]; then cp "$src" "$out"; else exit 8; fi +EOF +chmod +x "$STUBS2/wget" + +# helper: run a snippet in a subshell that sources the SUT with a controlled env +# usage: run_sourced [VAR=value ...] +run_sourced() { + local snippet="$1"; shift + local tmp_out; tmp_out="$(mktemp)" + env -i PATH="$STUBS2:/usr/bin:/bin" HOME="$HOME" SERVE="$SERVE" "$@" \ + bash -c "source '$SUT'; $snippet" >"$tmp_out" 2>&1 + RUN_STATUS=$? + RUN_OUTPUT="$(cat "$tmp_out")" + rm -f "$tmp_out" +} + +cp "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" +echo "zipbytes-metro" > "$SERVE/metro.zip" +echo "zipbytes-pierce" > "$SERVE/pierce.zip" +echo "1_M1 3_P1" > "$SERVE/StopConsolidation.txt" + +run_sourced "download_bundle_inputs && echo MAPPING=\$MAPPING_PATH" \ + BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -eq 0 ] && pass "download_bundle_inputs succeeds" || fail "download_bundle_inputs succeeds: $RUN_OUTPUT" +[ -f "$WORK2/inputs/metro.zip" ] && pass "metro.zip downloaded" || fail "metro.zip downloaded" +[ -f "$WORK2/inputs/pierce.zip" ] && pass "pierce.zip downloaded" || fail "pierce.zip downloaded" +[ -f "$WORK2/StopConsolidation.txt" ] && pass "mapping downloaded to StopConsolidation.txt" || fail "mapping downloaded to StopConsolidation.txt" +assert_contains "$RUN_OUTPUT" "MAPPING=$WORK2/StopConsolidation.txt" "MAPPING_PATH set" + +# no-mapping manifest → MAPPING_PATH empty +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +cp "$TESTDATA/bundle-inputs-no-mapping.json" "$SERVE/bundle-inputs.json" +run_sourced "download_bundle_inputs && echo MAPPING=[\$MAPPING_PATH]" \ + BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +assert_contains "$RUN_OUTPUT" "MAPPING=[]" "no stopConsolidationUrl -> empty MAPPING_PATH" +[ ! -e "$WORK2/StopConsolidation.txt" ] && pass "no mapping file created" || fail "no mapping file created" + +# unsupported version +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +echo '{"version": 2, "feeds": []}' > "$SERVE/bundle-inputs.json" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "unsupported manifest version fails" || fail "unsupported manifest version fails" +assert_contains "$RUN_OUTPUT" "ERROR: unsupported bundle-inputs version" "version error message" + +# empty feeds +echo '{"version": 1, "feeds": []}' > "$SERVE/bundle-inputs.json" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "empty feeds fails" || fail "empty feeds fails" +assert_contains "$RUN_OUTPUT" "ERROR: bundle-inputs manifest lists no feeds" "empty-feeds error message" + +# missing feed zip → ERROR naming the feed +cp "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" +rm -f "$SERVE/pierce.zip" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "missing feed download fails" || fail "missing feed download fails" +assert_contains "$RUN_OUTPUT" "ERROR: failed to download feed 'pierce'" "feed download error names the feed" +echo "zipbytes-pierce" > "$SERVE/pierce.zip" + +# sha256 mismatch → ERROR naming the feed +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +python3 - "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" <<'EOF' +import json, sys +m = json.load(open(sys.argv[1])) +m["feeds"][0]["sha256"] = "0" * 64 +json.dump(m, open(sys.argv[2], "w")) +EOF +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "sha256 mismatch fails" || fail "sha256 mismatch fails" +assert_contains "$RUN_OUTPUT" "ERROR: sha256 mismatch for feed 'metro'" "sha mismatch error names the feed" + +# sha256 match succeeds +GOOD_SHA="$(sha256sum "$SERVE/metro.zip" | cut -d' ' -f1)" +python3 - "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" "$GOOD_SHA" <<'EOF' +import json, sys +m = json.load(open(sys.argv[1])) +m["feeds"][0]["sha256"] = sys.argv[3] +json.dump(m, open(sys.argv[2], "w")) +EOF +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -eq 0 ] && pass "matching sha256 passes" || fail "matching sha256 passes: $RUN_OUTPUT" + +rm -rf "$WORK2" +# NOTE: $STUBS2 and $SERVE are intentionally NOT removed here — Task 4's tests +# reuse run_sourced (which references $STUBS2 in PATH). Cleanup happens at the +# end of the file once all sourced-function tests are done. + echo "" echo "==============================" echo "Results: $passed passed, $failed failed" diff --git a/bin/testdata/build_bundle/bundle-inputs-no-mapping.json b/bin/testdata/build_bundle/bundle-inputs-no-mapping.json new file mode 100644 index 0000000..e2e24ed --- /dev/null +++ b/bin/testdata/build_bundle/bundle-inputs-no-mapping.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "feeds": [ + {"id": "metro", "name": "King County Metro", "defaultAgencyId": "1", "url": "http://fixtures.test/metro.zip"}, + {"id": "pierce", "name": "Pierce Transit", "defaultAgencyId": "3", "url": "http://fixtures.test/pierce.zip"} + ] +} diff --git a/bin/testdata/build_bundle/bundle-inputs.json b/bin/testdata/build_bundle/bundle-inputs.json new file mode 100644 index 0000000..e14d05b --- /dev/null +++ b/bin/testdata/build_bundle/bundle-inputs.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "feeds": [ + {"id": "metro", "name": "King County Metro", "defaultAgencyId": "1", "url": "http://fixtures.test/metro.zip"}, + {"id": "pierce", "name": "Pierce Transit", "defaultAgencyId": "3", "url": "http://fixtures.test/pierce.zip"} + ], + "stopConsolidationUrl": "http://fixtures.test/StopConsolidation.txt" +} diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index b91d2f7..2a57bc9 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -85,6 +85,65 @@ fetch_url() { fi } +# Downloads manifest, feed zips, and optional mapping. Requires jq. +# Sets MAPPING_PATH to the downloaded mapping file path, or "" when absent. +MAPPING_PATH="" + +download_bundle_inputs() { + local inputs_dir="$BUNDLE_DIR/inputs" + local manifest="$inputs_dir/bundle-inputs.json" + mkdir -p "$inputs_dir" + + fetch_url "$BUNDLE_INPUTS_URL" "$manifest" "bundle-inputs manifest" + + local version + version="$(jq -r '.version' "$manifest")" + if [ "$version" != "1" ]; then + echo "ERROR: unsupported bundle-inputs version: ${version}" >&2 + exit 1 + fi + + local feed_count + feed_count="$(jq -r '.feeds | length' "$manifest")" + if [ "$feed_count" -eq 0 ]; then + echo "ERROR: bundle-inputs manifest lists no feeds" >&2 + exit 1 + fi + + local i id url sha dest + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + url="$(jq -r ".feeds[$i].url" "$manifest")" + sha="$(jq -r ".feeds[$i].sha256 // empty" "$manifest")" + dest="$inputs_dir/${id}.zip" + + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download feed '${id}' from ${url}" >&2 + exit 1 + fi + + if [ -n "$sha" ]; then + if ! echo "${sha} ${dest}" | sha256sum -c - > /dev/null 2>&1; then + echo "ERROR: sha256 mismatch for feed '${id}' (${dest})" >&2 + exit 1 + fi + fi + i=$((i + 1)) + done + + local mapping_url + mapping_url="$(jq -r '.stopConsolidationUrl // empty' "$manifest")" + if [ -n "$mapping_url" ]; then + # StopConsolidation.txt is the hardcoded filename ConsolidatedStopsServiceImpl + # reads from the bundle directory at runtime. + MAPPING_PATH="$BUNDLE_DIR/StopConsolidation.txt" + fetch_url "$mapping_url" "$MAPPING_PATH" "stop consolidation mapping" + else + MAPPING_PATH="" + fi +} + run_single_mode() { # Set default filename if using GTFS_URL if [ -n "$GTFS_URL" ]; then diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index b91d2f7..2a57bc9 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -85,6 +85,65 @@ fetch_url() { fi } +# Downloads manifest, feed zips, and optional mapping. Requires jq. +# Sets MAPPING_PATH to the downloaded mapping file path, or "" when absent. +MAPPING_PATH="" + +download_bundle_inputs() { + local inputs_dir="$BUNDLE_DIR/inputs" + local manifest="$inputs_dir/bundle-inputs.json" + mkdir -p "$inputs_dir" + + fetch_url "$BUNDLE_INPUTS_URL" "$manifest" "bundle-inputs manifest" + + local version + version="$(jq -r '.version' "$manifest")" + if [ "$version" != "1" ]; then + echo "ERROR: unsupported bundle-inputs version: ${version}" >&2 + exit 1 + fi + + local feed_count + feed_count="$(jq -r '.feeds | length' "$manifest")" + if [ "$feed_count" -eq 0 ]; then + echo "ERROR: bundle-inputs manifest lists no feeds" >&2 + exit 1 + fi + + local i id url sha dest + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + url="$(jq -r ".feeds[$i].url" "$manifest")" + sha="$(jq -r ".feeds[$i].sha256 // empty" "$manifest")" + dest="$inputs_dir/${id}.zip" + + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download feed '${id}' from ${url}" >&2 + exit 1 + fi + + if [ -n "$sha" ]; then + if ! echo "${sha} ${dest}" | sha256sum -c - > /dev/null 2>&1; then + echo "ERROR: sha256 mismatch for feed '${id}' (${dest})" >&2 + exit 1 + fi + fi + i=$((i + 1)) + done + + local mapping_url + mapping_url="$(jq -r '.stopConsolidationUrl // empty' "$manifest")" + if [ -n "$mapping_url" ]; then + # StopConsolidation.txt is the hardcoded filename ConsolidatedStopsServiceImpl + # reads from the bundle directory at runtime. + MAPPING_PATH="$BUNDLE_DIR/StopConsolidation.txt" + fetch_url "$mapping_url" "$MAPPING_PATH" "stop consolidation mapping" + else + MAPPING_PATH="" + fi +} + run_single_mode() { # Set default filename if using GTFS_URL if [ -n "$GTFS_URL" ]; then From 50322ddd8bde9bcd3a35df5776b70afa6245fc84 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 15:02:08 -0700 Subject: [PATCH 04/13] Fix run_sourced PATH so sha256sum resolves on macOS (/sbin) --- bin/build_bundle_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh index 0414bf5..67e4a44 100755 --- a/bin/build_bundle_test.sh +++ b/bin/build_bundle_test.sh @@ -152,7 +152,7 @@ chmod +x "$STUBS2/wget" run_sourced() { local snippet="$1"; shift local tmp_out; tmp_out="$(mktemp)" - env -i PATH="$STUBS2:/usr/bin:/bin" HOME="$HOME" SERVE="$SERVE" "$@" \ + env -i PATH="$STUBS2:/usr/bin:/bin:/usr/sbin:/sbin" HOME="$HOME" SERVE="$SERVE" "$@" \ bash -c "source '$SUT'; $snippet" >"$tmp_out" 2>&1 RUN_STATUS=$? RUN_OUTPUT="$(cat "$tmp_out")" From 7fad13b9178efe157c0819719f0ede472838a91e Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 15:06:43 -0700 Subject: [PATCH 05/13] Generate Spring bundle-context.xml from the bundle-inputs manifest --- bin/build_bundle_test.sh | 29 +++++++++ .../golden-context-no-mapping.xml | 20 +++++++ .../golden-context-with-mapping.xml | 29 +++++++++ bundler/build_bundle.sh | 59 +++++++++++++++++++ oba/build_bundle.sh | 59 +++++++++++++++++++ 5 files changed, 196 insertions(+) create mode 100644 bin/testdata/build_bundle/golden-context-no-mapping.xml create mode 100644 bin/testdata/build_bundle/golden-context-with-mapping.xml diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh index 67e4a44..f5e7afd 100755 --- a/bin/build_bundle_test.sh +++ b/bin/build_bundle_test.sh @@ -230,6 +230,35 @@ rm -rf "$WORK2" # reuse run_sourced (which references $STUBS2 in PATH). Cleanup happens at the # end of the file once all sourced-function tests are done. +# --- generate_bundle_context_xml ---------------------------------------------- + +WORK3="$(mktemp -d)" +mkdir -p "$WORK3/inputs" +cp "$TESTDATA/bundle-inputs.json" "$WORK3/inputs/bundle-inputs.json" + +run_sourced "generate_bundle_context_xml '$WORK3/inputs/bundle-inputs.json' '$WORK3/inputs' '$WORK3/StopConsolidation.txt' '$WORK3/bundle-context.xml'" \ + BUNDLE_DIR="$WORK3" +[ "$RUN_STATUS" -eq 0 ] && pass "xml generation (with mapping) succeeds" || fail "xml generation (with mapping) succeeds: $RUN_OUTPUT" + +sed -e "s|@INPUTS@|$WORK3/inputs|g" -e "s|@MAPPING@|$WORK3/StopConsolidation.txt|g" \ + "$TESTDATA/golden-context-with-mapping.xml" > "$WORK3/expected.xml" +if diff -u "$WORK3/expected.xml" "$WORK3/bundle-context.xml"; then + pass "context XML matches golden (with mapping)" +else + fail "context XML matches golden (with mapping)" +fi + +run_sourced "generate_bundle_context_xml '$WORK3/inputs/bundle-inputs.json' '$WORK3/inputs' '' '$WORK3/bundle-context-nm.xml'" \ + BUNDLE_DIR="$WORK3" +sed -e "s|@INPUTS@|$WORK3/inputs|g" "$TESTDATA/golden-context-no-mapping.xml" > "$WORK3/expected-nm.xml" +if diff -u "$WORK3/expected-nm.xml" "$WORK3/bundle-context-nm.xml"; then + pass "context XML matches golden (no mapping)" +else + fail "context XML matches golden (no mapping)" +fi + +rm -rf "$WORK3" + echo "" echo "==============================" echo "Results: $passed passed, $failed failed" diff --git a/bin/testdata/build_bundle/golden-context-no-mapping.xml b/bin/testdata/build_bundle/golden-context-no-mapping.xml new file mode 100644 index 0000000..5094667 --- /dev/null +++ b/bin/testdata/build_bundle/golden-context-no-mapping.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + diff --git a/bin/testdata/build_bundle/golden-context-with-mapping.xml b/bin/testdata/build_bundle/golden-context-with-mapping.xml new file mode 100644 index 0000000..791481e --- /dev/null +++ b/bin/testdata/build_bundle/golden-context-with-mapping.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index 2a57bc9..2d6d426 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -144,6 +144,65 @@ download_bundle_inputs() { fi } +# generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML +# Bean id "gtfs-bundles" and bean name "entityReplacementStrategy" are looked up +# by those exact names inside the federation builder — do not rename. +generate_bundle_context_xml() { + local manifest="$1" inputs_dir="$2" mapping_path="$3" out_xml="$4" + + { + cat <<'XMLHEAD' + + + + + + +XMLHEAD + + local feed_count i id agency + feed_count="$(jq -r '.feeds | length' "$manifest")" + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + agency="$(jq -r ".feeds[$i].defaultAgencyId" "$manifest")" + cat < + + + +XMLFEED + i=$((i + 1)) + done + + cat <<'XMLMID' + + + +XMLMID + + if [ -n "$mapping_path" ]; then + cat < + + + + + + + +XMLMAP + fi + + cat <<'XMLTAIL' + + +XMLTAIL + } > "$out_xml" +} + run_single_mode() { # Set default filename if using GTFS_URL if [ -n "$GTFS_URL" ]; then diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index 2a57bc9..2d6d426 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -144,6 +144,65 @@ download_bundle_inputs() { fi } +# generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML +# Bean id "gtfs-bundles" and bean name "entityReplacementStrategy" are looked up +# by those exact names inside the federation builder — do not rename. +generate_bundle_context_xml() { + local manifest="$1" inputs_dir="$2" mapping_path="$3" out_xml="$4" + + { + cat <<'XMLHEAD' + + + + + + +XMLHEAD + + local feed_count i id agency + feed_count="$(jq -r '.feeds | length' "$manifest")" + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + agency="$(jq -r ".feeds[$i].defaultAgencyId" "$manifest")" + cat < + + + +XMLFEED + i=$((i + 1)) + done + + cat <<'XMLMID' + + + +XMLMID + + if [ -n "$mapping_path" ]; then + cat < + + + + + + + +XMLMAP + fi + + cat <<'XMLTAIL' + + +XMLTAIL + } > "$out_xml" +} + run_single_mode() { # Set default filename if using GTFS_URL if [ -n "$GTFS_URL" ]; then From ffa52d89d4508884a842e85394881aeec756a24d Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 15:14:33 -0700 Subject: [PATCH 06/13] Implement multi-input mode orchestration --- bin/build_bundle_test.sh | 56 ++++++++++++++++++++++++++++++++++++++++ bundler/build_bundle.sh | 30 +++++++++++++++++++-- oba/build_bundle.sh | 30 +++++++++++++++++++-- 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh index f5e7afd..af21e07 100755 --- a/bin/build_bundle_test.sh +++ b/bin/build_bundle_test.sh @@ -259,6 +259,62 @@ fi rm -rf "$WORK3" +# --- run_multi_mode end-to-end (stubbed java) --------------------------------- + +STUBS3="$(mktemp -d)" +WORK4="$(mktemp -d)" +SERVE2="$(mktemp -d)" + +# The wget stub is re-declared locally (same behavior as Task 3's) because this +# block runs the full script as a subprocess with its own SERVE dir. +cat > "$STUBS3/wget" <<'EOF' +#!/bin/bash +out="" url="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else url="$1"; shift; fi +done +src="$SERVE/$(basename "$url")" +if [ -f "$src" ]; then cp "$src" "$out"; else exit 8; fi +EOF +cat > "$STUBS3/java" <<'EOF' +#!/bin/bash +echo "java $*" >> "$STUB_LOG" +echo "cwd $PWD" >> "$STUB_LOG" +EOF +cat > "$STUBS3/gtfstidy" <<'EOF' +#!/bin/bash +echo "gtfstidy WAS CALLED" >> "$STUB_LOG" +EOF +chmod +x "$STUBS3/wget" "$STUBS3/java" "$STUBS3/gtfstidy" + +cp "$TESTDATA/bundle-inputs.json" "$SERVE2/bundle-inputs.json" +echo "zipbytes-metro" > "$SERVE2/metro.zip" +echo "zipbytes-pierce" > "$SERVE2/pierce.zip" +echo "1_M1 3_P1" > "$SERVE2/StopConsolidation.txt" + +STUB_LOG="$WORK4/stub.log"; : > "$STUB_LOG" + +env -i PATH="$STUBS3:/usr/bin:/bin" HOME="$HOME" SERVE="$SERVE2" STUB_LOG="$STUB_LOG" \ + BUNDLE_DIR="$WORK4" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json \ + TDF_BUILDER_JAR=/fake/builder.jar \ + bash "$SUT" > "$WORK4/run.out" 2>&1 +MULTI_STATUS=$? +MULTI_OUTPUT="$(cat "$WORK4/run.out")" + +[ "$MULTI_STATUS" -eq 0 ] && pass "multi mode (stubbed) exits 0" || fail "multi mode (stubbed) exits 0: $MULTI_OUTPUT" +assert_contains "$(cat "$STUB_LOG")" "java -Xss4m -Xmx3g -jar /fake/builder.jar bundle-context.xml ." "multi mode builder argv" +assert_contains "$(cat "$STUB_LOG")" "cwd $WORK4" "multi mode builder cwd is bundle dir" +if grep -q "gtfstidy WAS CALLED" "$STUB_LOG"; then + fail "multi mode must not run gtfstidy" +else + pass "multi mode must not run gtfstidy" +fi +[ -f "$WORK4/bundle-context.xml" ] && pass "bundle-context.xml written to bundle dir" || fail "bundle-context.xml written to bundle dir" +[ -f "$WORK4/StopConsolidation.txt" ] && pass "StopConsolidation.txt present in bundle output dir" || fail "StopConsolidation.txt present in bundle output dir" +assert_contains "$MULTI_OUTPUT" "Multi-input mode" "multi mode banner printed" + +rm -rf "$STUBS3" "$WORK4" "$SERVE2" + echo "" echo "==============================" echo "Results: $passed passed, $failed failed" diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index 2d6d426..a0b2833 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -249,8 +249,34 @@ run_single_mode() { run_multi_mode() { echo "OBA Bundle Builder Starting" echo "Multi-input mode: BUNDLE_INPUTS_URL: $BUNDLE_INPUTS_URL" - echo "ERROR: multi-input mode not yet implemented" >&2 - exit 1 + echo "OBA Version: $OBA_VERSION" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + echo "gtfstidy: skipped in multi-input mode (parity with the legacy multi-feed build)" + + mkdir -p "$BUNDLE_DIR" + cd "$BUNDLE_DIR" + + download_bundle_inputs + + generate_bundle_context_xml \ + "$BUNDLE_DIR/inputs/bundle-inputs.json" \ + "$BUNDLE_DIR/inputs" \ + "$MAPPING_PATH" \ + "$BUNDLE_DIR/bundle-context.xml" + + # Any non-.zip, non-directory arg before the last is treated as a Spring + # context path by FederatedTransitDataBundleCreatorMain; "." is the output dir. + # The JAR must be executed from within the same directory as the bundle, + # or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" bundle-context.xml . + + # MAPPING_PATH already lives at $BUNDLE_DIR/StopConsolidation.txt — the + # hardcoded name ConsolidatedStopsServiceImpl reads at runtime. Assert it + # survived the build rather than trusting the download. + if [ -n "$MAPPING_PATH" ] && [ ! -f "$MAPPING_PATH" ]; then + echo "ERROR: stop consolidation mapping missing from bundle output dir after build" >&2 + exit 1 + fi } main() { diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index 2d6d426..a0b2833 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -249,8 +249,34 @@ run_single_mode() { run_multi_mode() { echo "OBA Bundle Builder Starting" echo "Multi-input mode: BUNDLE_INPUTS_URL: $BUNDLE_INPUTS_URL" - echo "ERROR: multi-input mode not yet implemented" >&2 - exit 1 + echo "OBA Version: $OBA_VERSION" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + echo "gtfstidy: skipped in multi-input mode (parity with the legacy multi-feed build)" + + mkdir -p "$BUNDLE_DIR" + cd "$BUNDLE_DIR" + + download_bundle_inputs + + generate_bundle_context_xml \ + "$BUNDLE_DIR/inputs/bundle-inputs.json" \ + "$BUNDLE_DIR/inputs" \ + "$MAPPING_PATH" \ + "$BUNDLE_DIR/bundle-context.xml" + + # Any non-.zip, non-directory arg before the last is treated as a Spring + # context path by FederatedTransitDataBundleCreatorMain; "." is the output dir. + # The JAR must be executed from within the same directory as the bundle, + # or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" bundle-context.xml . + + # MAPPING_PATH already lives at $BUNDLE_DIR/StopConsolidation.txt — the + # hardcoded name ConsolidatedStopsServiceImpl reads at runtime. Assert it + # survived the build rather than trusting the download. + if [ -n "$MAPPING_PATH" ] && [ ! -f "$MAPPING_PATH" ]; then + echo "ERROR: stop consolidation mapping missing from bundle output dir after build" >&2 + exit 1 + fi } main() { From de465b1df3d8512a19b444fee8c27269cf6950ee Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 15:21:52 -0700 Subject: [PATCH 07/13] Support standalone STOP_CONSOLIDATION_URL in single-zip mode --- bin/build_bundle_test.sh | 49 ++++++++++++++++++++++++++++++++++++++++ bundler/build_bundle.sh | 30 +++++++++++++++++++++++- oba/build_bundle.sh | 30 +++++++++++++++++++++++- 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh index af21e07..137c21b 100755 --- a/bin/build_bundle_test.sh +++ b/bin/build_bundle_test.sh @@ -315,6 +315,55 @@ assert_contains "$MULTI_OUTPUT" "Multi-input mode" "multi mode banner printed" rm -rf "$STUBS3" "$WORK4" "$SERVE2" +# --- single mode + STOP_CONSOLIDATION_URL -------------------------------------- + +STUBS4="$(mktemp -d)" +WORK5="$(mktemp -d)" +SERVE3="$(mktemp -d)" + +cat > "$STUBS4/wget" <<'EOF' +#!/bin/bash +out="" url="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else url="$1"; shift; fi +done +src="$SERVE/$(basename "$url")" +if [ -f "$src" ]; then cp "$src" "$out"; else exit 8; fi +EOF +cat > "$STUBS4/gtfstidy" <<'EOF' +#!/bin/bash +exit 0 +EOF +cat > "$STUBS4/java" <<'EOF' +#!/bin/bash +echo "java $*" >> "$STUB_LOG" +EOF +chmod +x "$STUBS4/wget" "$STUBS4/gtfstidy" "$STUBS4/java" + +echo "fake-gtfs" > "$SERVE3/gtfs.zip" +echo "1_M1 3_P1" > "$SERVE3/StopConsolidation.txt" +STUB_LOG="$WORK5/stub.log"; : > "$STUB_LOG" + +env -i PATH="$STUBS4:/usr/bin:/bin" HOME="$HOME" SERVE="$SERVE3" STUB_LOG="$STUB_LOG" \ + BUNDLE_DIR="$WORK5" GTFS_URL=http://fixtures.test/gtfs.zip \ + STOP_CONSOLIDATION_URL=http://fixtures.test/StopConsolidation.txt \ + TDF_BUILDER_JAR=/fake/builder.jar \ + bash "$SUT" > "$WORK5/run.out" 2>&1 +SC_STATUS=$? + +[ "$SC_STATUS" -eq 0 ] && pass "single mode + consolidation exits 0" || fail "single mode + consolidation exits 0: $(cat "$WORK5/run.out")" +[ -f "$WORK5/StopConsolidation.txt" ] && pass "single mode downloads mapping" || fail "single mode downloads mapping" +[ -f "$WORK5/consolidation-context.xml" ] && pass "single mode writes consolidation-context.xml" || fail "single mode writes consolidation-context.xml" +assert_contains "$(cat "$STUB_LOG")" "java -Xss4m -Xmx3g -jar /fake/builder.jar ./gtfs_pristine.zip consolidation-context.xml ." "single mode consolidation builder argv" +if grep -q "entityReplacementStrategy" "$WORK5/consolidation-context.xml" && ! grep -q "gtfs-bundles" "$WORK5/consolidation-context.xml"; then + pass "consolidation-context.xml has replacement beans only" +else + fail "consolidation-context.xml has replacement beans only" +fi + +rm -rf "$STUBS4" "$WORK5" "$SERVE3" +rm -rf "$STUBS2" "$SERVE" # deferred cleanup from the Task 3/4 sourced-function tests + echo "" echo "==============================" echo "Results: $passed passed, $failed failed" diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index a0b2833..3fa9b68 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -203,6 +203,27 @@ XMLTAIL } > "$out_xml" } +# generate_single_mode_context_xml MAPPING_PATH OUT_XML — replacement beans only. +generate_single_mode_context_xml() { + local mapping_path="$1" out_xml="$2" + cat > "$out_xml" < + + + + + + + + + + + + +XMLSC +} + run_single_mode() { # Set default filename if using GTFS_URL if [ -n "$GTFS_URL" ]; then @@ -241,9 +262,16 @@ run_single_mode() { GTFS_ZIP_FILENAME="gtfs_tidied.zip" fi + local context_args=() + if [ -n "$STOP_CONSOLIDATION_URL" ]; then + fetch_url "$STOP_CONSOLIDATION_URL" "$BUNDLE_DIR/StopConsolidation.txt" "stop consolidation mapping" + generate_single_mode_context_xml "$BUNDLE_DIR/StopConsolidation.txt" "$BUNDLE_DIR/consolidation-context.xml" + context_args=(consolidation-context.xml) + fi + # The JAR must be executed from within the same directory # as the bundle, or else some necessary files are not generated. - java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" . + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" ${context_args[@]+"${context_args[@]}"} . } run_multi_mode() { diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index a0b2833..3fa9b68 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -203,6 +203,27 @@ XMLTAIL } > "$out_xml" } +# generate_single_mode_context_xml MAPPING_PATH OUT_XML — replacement beans only. +generate_single_mode_context_xml() { + local mapping_path="$1" out_xml="$2" + cat > "$out_xml" < + + + + + + + + + + + + +XMLSC +} + run_single_mode() { # Set default filename if using GTFS_URL if [ -n "$GTFS_URL" ]; then @@ -241,9 +262,16 @@ run_single_mode() { GTFS_ZIP_FILENAME="gtfs_tidied.zip" fi + local context_args=() + if [ -n "$STOP_CONSOLIDATION_URL" ]; then + fetch_url "$STOP_CONSOLIDATION_URL" "$BUNDLE_DIR/StopConsolidation.txt" "stop consolidation mapping" + generate_single_mode_context_xml "$BUNDLE_DIR/StopConsolidation.txt" "$BUNDLE_DIR/consolidation-context.xml" + context_args=(consolidation-context.xml) + fi + # The JAR must be executed from within the same directory # as the bundle, or else some necessary files are not generated. - java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" . + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" ${context_args[@]+"${context_args[@]}"} . } run_multi_mode() { From 1b7c34cf098aeff3b2ad87a2556ad5b4a95d86a6 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 15:29:38 -0700 Subject: [PATCH 08/13] Wire ConsolidatedStopsModificationStrategy into GTFS-RT sources --- ...ata-federation-webapp-data-sources.xml.hbs | 16 +++++++ oba/config/template_renderer/main_test.go | 46 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs b/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs index 0ca4708..ec13377 100644 --- a/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs +++ b/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs @@ -77,6 +77,22 @@ {{/if}} + {{#if this.agencyIds.length}} + + + + {{#each this.agencyIds}}{{#if @first}}{{/if}}{{/each}} + + + {{else if this.agencyId}} + + + + + + {{/if}} + {{#if this.feedApiKey}} diff --git a/oba/config/template_renderer/main_test.go b/oba/config/template_renderer/main_test.go index ceba2dc..c702b44 100644 --- a/oba/config/template_renderer/main_test.go +++ b/oba/config/template_renderer/main_test.go @@ -217,3 +217,49 @@ func TestFederationTemplateSingularAgencyId(t *testing.T) { t.Errorf("did not expect plural agencyIds list:\n%s", out) } } + +func TestFederationTemplateStopModificationStrategy(t *testing.T) { + json := `{"FEEDS":[` + + `{"tripUpdatesUrl":"https://a/trips","agencyIds":["1","40"]},` + + `{"vehiclePositionsUrl":"https://b/vehicles","agencyId":"3"},` + + `{"alertsUrl":"https://c/alerts"}` + + `]}` + + out, err := renderTemplate(federationTemplatePath, json) + if err != nil { + t.Fatalf("renderTemplate returned an error: %v", err) + } + // Feeds with an agency id (list or singular) get the strategy; the + // agency-less feed must not. + if c := strings.Count(out, "ConsolidatedStopsModificationStrategy"); c != 2 { + t.Errorf("expected 2 strategy beans, got %d\n%s", c, out) + } + if c := strings.Count(out, ``); c != 2 { + t.Errorf("expected 2 stopModificationStrategy properties, got %d\n%s", c, out) + } + // Multi-agency feed: the strategy is namespaced to the FIRST listed agency. + // (Feed 1 uses the agencyIds list form, so the only `agencyId` property it + // renders is the strategy's.) + if !strings.Contains(out, ``) { + t.Errorf("strategy for multi-agency feed should use first agency:\n%s", out) + } + if strings.Contains(out, ``) { + t.Errorf("strategy must not be generated for non-first agencies:\n%s", out) + } +} + +func TestFederationTemplateStopModificationStrategySingularAgency(t *testing.T) { + json := `{"FEEDS":[{"tripUpdatesUrl":"https://x/trips","agencyId":"unitrans"}]}` + out, err := renderTemplate(federationTemplatePath, json) + if err != nil { + t.Fatalf("renderTemplate returned an error: %v", err) + } + if c := strings.Count(out, "ConsolidatedStopsModificationStrategy"); c != 1 { + t.Errorf("expected 1 strategy bean, got %d\n%s", c, out) + } + // Singular form renders agencyId twice: once on the GtfsRealtimeSource, + // once on the strategy bean. + if c := strings.Count(out, ``); c != 2 { + t.Errorf("expected agencyId on source and strategy, got %d\n%s", c, out) + } +} From c9efa3bab149463967b1345dbb0f3e56d45b4722 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 15:51:54 -0700 Subject: [PATCH 09/13] Add jq to bundler image and CI job for build_bundle.sh tests --- .github/workflows/test.yaml | 15 +++++++++++++++ bundler/Dockerfile | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6a321e7..2f3de32 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,6 +25,21 @@ jobs: working-directory: oba/config/template_renderer run: go test ./... + bundle_script: + name: build_bundle.sh unit tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Guard - oba and bundler scripts identical + run: diff oba/build_bundle.sh bundler/build_bundle.sh + + - name: Run script tests + run: bash bin/build_bundle_test.sh + image: name: Build Docker Image runs-on: ubuntu-latest diff --git a/bundler/Dockerfile b/bundler/Dockerfile index 48dcc85..672ca23 100644 --- a/bundler/Dockerfile +++ b/bundler/Dockerfile @@ -42,7 +42,7 @@ FROM tomcat:8.5.100-jdk11-temurin ARG OBA_VERSION ENV OBA_VERSION=${OBA_VERSION} -RUN apt-get update && apt-get install -y unzip zip +RUN apt-get update && apt-get install -y jq unzip zip # Start configuring OBA WORKDIR /oba From 8ad526e2099b6d21fa26506920f68d244037e899 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 16:05:06 -0700 Subject: [PATCH 10/13] Integration test: real builder with bundle inputs and consolidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs the real transit-data-federation builder jar inside the built oba image against two 3-stop fixture GTFS feeds in multi-input mode, proving the manifest/sha256/entity-replacement path end-to-end rather than through the stubbed unit harness alone. Also pins the keeper-missing mapping-row failure mode: the replaced stop is dropped from the bundle either way, but with no valid keeper the builder leaves a dangling stopTime pointing at a null stop (StopTimeEntriesFactory ERROR) instead of cleanly remapping it — the concrete harm that justifies obacloud excluding keeper-missing rows before publish. Assertions were adjusted from the plan's draft based on what the real jar actually does: the hypothesized "error replacing entity ... replacement not found" log line does not appear against the 2.7.1 builder, and an anchored `strings | grep '^ID$'` probe against the serialized TransitGraph.obj is flaky because the Java serialization stream has no line terminators. Both were replaced with directly verified, reproducible checks. --- .github/workflows/test.yaml | 20 ++++ bin/bundle_inputs_integration_test.sh | 113 ++++++++++++++++++ bin/testdata/build_bundle/make_fixtures.sh | 45 +++++++ .../build_bundle/mapping-keeper-missing.txt | 2 + bin/testdata/build_bundle/mapping-valid.txt | 2 + 5 files changed, 182 insertions(+) create mode 100755 bin/bundle_inputs_integration_test.sh create mode 100755 bin/testdata/build_bundle/make_fixtures.sh create mode 100644 bin/testdata/build_bundle/mapping-keeper-missing.txt create mode 100644 bin/testdata/build_bundle/mapping-valid.txt diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2f3de32..cf8f1be 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,6 +40,26 @@ jobs: - name: Run script tests run: bash bin/build_bundle_test.sh + bundle_inputs_integration: + name: Bundle inputs integration (real builder) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Build oba image + run: | + docker buildx build \ + --load \ + -t oba-image:latest \ + ./oba + + - name: Run integration test + timeout-minutes: 20 + run: | + chmod +x bin/bundle_inputs_integration_test.sh bin/testdata/build_bundle/make_fixtures.sh + bash bin/bundle_inputs_integration_test.sh + image: name: Build Docker Image runs-on: ubuntu-latest diff --git a/bin/bundle_inputs_integration_test.sh b/bin/bundle_inputs_integration_test.sh new file mode 100755 index 0000000..782b998 --- /dev/null +++ b/bin/bundle_inputs_integration_test.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Integration test: runs the REAL federation builder inside the built oba image +# in multi-input mode against fixture feeds. Requires: docker, python3, an +# `oba-image:latest` local image (build with: +# docker buildx build --load -t oba-image:latest ./oba ). +# Linux-first (uses --network host for the fixture HTTP server); on macOS run +# in CI instead. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +IMAGE="${OBA_IMAGE:-oba-image:latest}" +PORT=8123 + +passed=0 +failed=0 +pass() { echo "PASS: $1"; passed=$((passed + 1)); } +fail() { echo "FAIL: $1"; failed=$((failed + 1)); } + +SERVE="$(mktemp -d)" +OUT_VALID="$(mktemp -d)" +OUT_KM="$(mktemp -d)" + +bash "$REPO_ROOT/bin/testdata/build_bundle/make_fixtures.sh" "$SERVE" +cp "$REPO_ROOT/bin/testdata/build_bundle/mapping-valid.txt" "$SERVE/StopConsolidation.txt" + +cat > "$SERVE/bundle-inputs.json" </dev/null 2>&1 & +SERVER_PID=$! +trap 'kill $SERVER_PID 2>/dev/null; rm -rf "$SERVE" "$OUT_VALID" "$OUT_KM"' EXIT +sleep 1 + +run_build() { + # --user matches the host user so the mounted output dir stays deletable + # by the test's cleanup trap (files written as root would survive rm -rf + # on CI runners). + local out_dir="$1" + docker run --rm --network host \ + --user "$(id -u):$(id -g)" \ + -v "$out_dir":/bundle \ + -e BUNDLE_INPUTS_URL="http://127.0.0.1:${PORT}/bundle-inputs.json" \ + --entrypoint bash \ + "$IMAGE" /oba/build_bundle.sh +} + +echo "--- Valid mapping build ---" +if run_build "$OUT_VALID" > "$OUT_VALID/build.log" 2>&1; then + pass "multi-input build with mapping exits 0" +else + fail "multi-input build with mapping exits 0" + tail -50 "$OUT_VALID/build.log" +fi + +[ -f "$OUT_VALID/TransitGraph.obj" ] && pass "bundle artifact TransitGraph.obj built" || fail "bundle artifact TransitGraph.obj built" +[ -f "$OUT_VALID/StopConsolidation.txt" ] && pass "StopConsolidation.txt in bundle output" || fail "StopConsolidation.txt in bundle output" +grep -q "gtfs=/bundle/inputs/metro.zip" "$OUT_VALID/build.log" && pass "metro loaded as its own bundle" || fail "metro loaded as its own bundle" +grep -q "gtfs=/bundle/inputs/pierce.zip" "$OUT_VALID/build.log" && pass "pierce loaded as its own bundle" || fail "pierce loaded as its own bundle" + +# Replaced stop is gone, keeper survives. TransitGraph.obj is one continuous +# Java serialization stream (TC_STRING tag + 2-byte length prefix + bytes, +# no line terminator), so `strings` glues "M2" to whatever stream token +# follows (observed: "M2pq") and an anchored `^M2$` line match is flaky — +# it happened to pass for the absence check and fail for the presence +# check on the same file. A direct byte-level substring grep on the raw +# file is reliable here because M2/P2 are short, unique 2-character ids +# that cannot occur elsewhere in the bundle's ASCII content (confirmed via +# `grep -aob "M2" TransitGraph.obj`: the sole hit is immediately preceded by +# the TC_STRING tag and a length=2 prefix, i.e. it is the serialized stop id +# field, not a coincidental substring). +grep -qa "P2" "$OUT_VALID/TransitGraph.obj" \ + && fail "replaced stop P2 absent from TransitGraph.obj" \ + || pass "replaced stop P2 absent from TransitGraph.obj" +grep -qa "M2" "$OUT_VALID/TransitGraph.obj" && pass "keeper M2 present in TransitGraph.obj" || fail "keeper M2 present in TransitGraph.obj" + +echo "--- Keeper-missing mapping build (behavior pin) ---" +cp "$REPO_ROOT/bin/testdata/build_bundle/mapping-keeper-missing.txt" "$SERVE/StopConsolidation.txt" +set +e +run_build "$OUT_KM" > "$OUT_KM/build.log" 2>&1 +KM_STATUS=$? +set -e +# Pin the observed behavior. "error replacing entity ... replacement not +# found" (hypothesized from reading app-modules source while writing this +# plan) does not appear against the real 2.7.1 builder jar — running this +# test is what surfaced that. What actually happens, verified by running +# it: the replaced stop (3_P2) is dropped from the graph exactly as in the +# valid-mapping build above, but nothing repoints the stopTime row that +# referenced it, so StopTimeEntriesFactory logs an ERROR for a stopTime +# with a null stop. That dangling reference — not a clean "replacement not +# found" warning — is the concrete harm that justifies obacloud excluding +# keeper-missing rows at publish. +grep -q "found stopTime without a stop id" "$OUT_KM/build.log" \ + && pass "keeper-missing line produces a dangling stopTime error" \ + || fail "keeper-missing line produces a dangling stopTime error" +grep -qa "P2" "$OUT_KM/TransitGraph.obj" \ + && fail "keeper-missing replaced stop P2 still absent from TransitGraph.obj" \ + || pass "keeper-missing replaced stop P2 still absent from TransitGraph.obj" +echo "keeper-missing build exit status: $KM_STATUS (recorded, not asserted — obacloud excludes such rows at publish)" + +echo "" +echo "==============================" +echo "Results: $passed passed, $failed failed" +echo "==============================" +[ "$failed" -eq 0 ] diff --git a/bin/testdata/build_bundle/make_fixtures.sh b/bin/testdata/build_bundle/make_fixtures.sh new file mode 100755 index 0000000..e92d3dc --- /dev/null +++ b/bin/testdata/build_bundle/make_fixtures.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Builds two 3-stop fixture GTFS zips (metro: agency 1, pierce: agency 3) +# into the directory given as $1. Stops M2/P2 sit at identical coordinates +# so consolidating "1_M2 3_P2" is geometrically sensible. +set -euo pipefail +OUT="${1:?usage: make_fixtures.sh OUT_DIR}" +mkdir -p "$OUT" + +make_feed() { + local dir="$1" agency_id="$2" agency_name="$3" prefix="$4" + mkdir -p "$dir" + cat > "$dir/agency.txt" < "$dir/stops.txt" < "$dir/routes.txt" < "$dir/calendar.txt" < "$dir/trips.txt" < "$dir/stop_times.txt" < Date: Sun, 5 Jul 2026 16:23:09 -0700 Subject: [PATCH 11/13] Document BUNDLE_INPUTS_URL and STOP_CONSOLIDATION_URL in README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5f89bfd..1f6f6a5 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ You can find the latest published Docker images on Docker Hub: * `JDBC_PASSWORD` - The password for your database. * GTFS (Optional, required only when using `oba_app` independently) * `GTFS_URL` - The URL to the GTFS feed you want to use. + * `BUNDLE_INPUTS_URL` - URL to a bundle-inputs.json manifest for multi-input mode; specifies per-feed zips, agency IDs, and optional stop consolidation mapping. Mutually exclusive with GTFS_URL, GTFS_ZIP_FILENAME, and STOP_CONSOLIDATION_URL. + * `STOP_CONSOLIDATION_URL` - (Single-zip mode only) URL to a stop-consolidation mapping file applied during bundle build. Mutually exclusive with BUNDLE_INPUTS_URL. * GTFS-RT Support (Optional) * `TZ` - The timezone for the server. Ensure that the server's timezone matches the timezone specified in your static GTFS `agency.txt` file. The timezone format is the IANA standard, and [a full list of timezones can be found on Wikipedia](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). * `GTFS_RT_FEEDS` - Preferred for configuring one OR many GTFS-RT feeds. A JSON array of feed objects; each object may contain `tripUpdatesUrl`, `vehiclePositionsUrl`, `alertsUrl`, `refreshInterval`, `agencyIds` (array), `feedApiKey`, and `feedApiValue`. Example: `[{"tripUpdatesUrl":"https://a/trips","agencyIds":["unitrans"]},{"vehiclePositionsUrl":"https://b/veh","agencyIds":["kcm"]}]`. When set, it takes precedence over the single-feed variables below. From 1b92aaa8dd6b79019cf7656408941aba66fa50aa Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 23:25:59 -0700 Subject: [PATCH 12/13] Pin Services-with-Bundler CI to a fixture; address review feedback The 'Services with Bundler' job was failing on stops-for-location returning 0 stops. Root cause is environmental, not this PR: the job built from the live Unitrans feed, and the current feed leaves ~42% of its stops out of OBA's runtime geospatial STRtree (near-coincident directional stop pairs). validate.sh picks 'first route's first stop' dynamically, so it fails at random depending on which stop it lands on. The single-mode build path, OBA JAR, and gtfstidy output are all unchanged by this PR (verified), so main would fail the same way today. Make the job deterministic by building from a small committed GTFS fixture (7 well-separated stops, all indexed) served on the compose network via a new docker-compose.ci.yml override, instead of the live feed. Also address CodeRabbit review feedback: - Escape manifest-derived ids/agencyIds before writing bundle-context.xml (new xml_attr_escape helper) + a unit test; both build_bundle.sh copies stay byte-identical. - zip ../gtfs_tidied.zip ./* (glob safety). - bundler/Dockerfile: --no-install-recommends + clean apt lists. - bundle_inputs_integration job: add contents:read permissions; persist-credentials:false on the two new jobs' checkouts. - Integration test: poll for the fixture server instead of a fixed sleep. --- .github/workflows/test.yaml | 11 +++ bin/build_bundle_test.sh | 9 +++ bin/bundle_inputs_integration_test.sh | 8 ++- .../services/make_services_fixture.sh | 64 ++++++++++++++++++ bin/testdata/services/services-gtfs.zip | Bin 0 -> 1495 bytes bundler/Dockerfile | 3 +- bundler/build_bundle.sh | 20 +++++- docker-compose.ci.yml | 42 ++++++++++++ oba/build_bundle.sh | 20 +++++- 9 files changed, 169 insertions(+), 8 deletions(-) create mode 100755 bin/testdata/services/make_services_fixture.sh create mode 100644 bin/testdata/services/services-gtfs.zip create mode 100644 docker-compose.ci.yml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cf8f1be..096ee27 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,6 +33,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Guard - oba and bundler scripts identical run: diff oba/build_bundle.sh bundler/build_bundle.sh @@ -43,9 +45,13 @@ jobs: bundle_inputs_integration: name: Bundle inputs integration (real builder) runs-on: ubuntu-latest + permissions: + contents: read steps: - name: Checkout code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build oba image run: | @@ -87,6 +93,11 @@ jobs: services: name: Services with Bundler runs-on: ubuntu-latest + # Build against a small committed GTFS fixture served on the compose network + # instead of the live Unitrans feed, so this job is deterministic. See + # docker-compose.ci.yml and bin/testdata/services/make_services_fixture.sh. + env: + COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh index 137c21b..fca1b23 100755 --- a/bin/build_bundle_test.sh +++ b/bin/build_bundle_test.sh @@ -257,6 +257,15 @@ else fail "context XML matches golden (no mapping)" fi +# XML metacharacters in manifest-derived ids/agency ids must be escaped, not +# emitted raw (which would produce malformed bundle-context.xml). +cat > "$WORK3/inputs/special.json" <<'EOF' +{"version": 1, "feeds": [{"id": "metro", "name": "n", "defaultAgencyId": "A&B/dev/null 2>&1 & SERVER_PID=$! trap 'kill $SERVER_PID 2>/dev/null; rm -rf "$SERVE" "$OUT_VALID" "$OUT_KM"' EXIT -sleep 1 + +# Poll until the fixture server answers, rather than a fixed sleep that can race +# on a slow/loaded runner (the in-container wget calls would fail if it isn't up). +for _ in $(seq 1 40); do + curl -sf "http://127.0.0.1:${PORT}/bundle-inputs.json" >/dev/null 2>&1 && break + sleep 0.25 +done run_build() { # --user matches the host user so the mounted output dir stays deletable diff --git a/bin/testdata/services/make_services_fixture.sh b/bin/testdata/services/make_services_fixture.sh new file mode 100755 index 0000000..d93bc81 --- /dev/null +++ b/bin/testdata/services/make_services_fixture.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Builds services-gtfs.zip: a small, self-contained GTFS feed used by the +# "Services with Bundler" CI job so that job is deterministic and does NOT +# depend on the live Unitrans feed (which drifts and, as of mid-2026, leaves +# ~40% of its stops out of OBA's geospatial index, making bin/validate.sh's +# stops-for-location check fail at random). +# +# Design notes for validate.sh compatibility: +# * one agency, one route, one trip, seven well-separated stops (~1 km apart) +# so every stop lands in OBA's geospatial STRtree (stops-for-location). +# * calendar spans a wide date range so agencies-with-coverage is populated. +# +# Regenerate with: bash bin/testdata/services/make_services_fixture.sh +set -euo pipefail +OUT_DIR="$(cd "$(dirname "$0")" && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/agency.txt" <<'EOF' +agency_id,agency_name,agency_url,agency_timezone +oba-test,OBA Test Transit,https://example.org,America/Los_Angeles +EOF + +# Seven stops strung ~1 km apart along a NE line near Davis, CA. Wide spacing +# keeps each stop distinct in the geospatial index. +cat > "$WORK/stops.txt" <<'EOF' +stop_id,stop_name,stop_lat,stop_lon +S1,First & Main,38.540000,-121.740000 +S2,Second & Oak,38.548000,-121.730000 +S3,Third & Elm,38.556000,-121.720000 +S4,Fourth & Pine,38.564000,-121.710000 +S5,Fifth & Cedar,38.572000,-121.700000 +S6,Sixth & Birch,38.580000,-121.690000 +S7,Seventh & Ash,38.588000,-121.680000 +EOF + +cat > "$WORK/routes.txt" <<'EOF' +route_id,agency_id,route_short_name,route_long_name,route_type +R1,oba-test,1,Test Line,3 +EOF + +cat > "$WORK/calendar.txt" <<'EOF' +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +WEEK,1,1,1,1,1,1,1,20200101,20401231 +EOF + +cat > "$WORK/trips.txt" <<'EOF' +route_id,service_id,trip_id +R1,WEEK,T1 +EOF + +cat > "$WORK/stop_times.txt" <<'EOF' +trip_id,arrival_time,departure_time,stop_id,stop_sequence +T1,08:00:00,08:00:00,S1,1 +T1,08:05:00,08:05:00,S2,2 +T1,08:10:00,08:10:00,S3,3 +T1,08:15:00,08:15:00,S4,4 +T1,08:20:00,08:20:00,S5,5 +T1,08:25:00,08:25:00,S6,6 +T1,08:30:00,08:30:00,S7,7 +EOF + +(cd "$WORK" && zip -q "$OUT_DIR/services-gtfs.zip" ./*.txt) +echo "wrote $OUT_DIR/services-gtfs.zip" diff --git a/bin/testdata/services/services-gtfs.zip b/bin/testdata/services/services-gtfs.zip new file mode 100644 index 0000000000000000000000000000000000000000..4a1de2e4d70acfdf762eb91f2f1c09aef07d4861 GIT binary patch literal 1495 zcmWIWW@Zs#U|`^2DBtxorr*u=WCD;^48&XvG7O37sd>qjdL3@;#)*c zoDB^-FDRL$7l7hM?gf!aQ9$?A0x=KJjme2Qsd*`hMTA@#dfJe$L4l|B;oZ$88ZSg= z*fc0BZgXQZ%UC^RD&#c1E zzJ-1q6M{aRJmY=U@EH>WszYu!X!e%?9q|E(`G5gcT#{c9Uy_+i*fFl%-du+bcv!z% zo{s(@tiJSm!1seDQ4+sAMVCMK->AdI)GOgTb(3zYtC;5fKM`4#Mq&3hU2%>$zjxnl z|J&B}e06s#cU0~;eRb#kKaDnfW~MA-WNpdYW6C8Wcs63OgYvPwCR45{6V3qH!prg& znR1yV%lV@n6hjBuMgnpFH8Q&&=o()9@Vo7 zb&|hSfSy$cVvuJ`iZTlbdKR8wH2rk7PW!MKdv_UeH<*A@$BxyjxBGP{iUZRFBa<96 zuIw!Vw2pxR7{Clm8bK`N%+3nQ>}VMs-8|H+h-}^xpn0enlQmy zL(K!o=A8wahn^#dGZQs!BAeOC1ThmDdBhoq8vDq`l`%6Qrv+Tbv9f_;j|~W)0R40q IsEL6A08l;8(*OVf literal 0 HcmV?d00001 diff --git a/bundler/Dockerfile b/bundler/Dockerfile index 672ca23..b294a03 100644 --- a/bundler/Dockerfile +++ b/bundler/Dockerfile @@ -42,7 +42,8 @@ FROM tomcat:8.5.100-jdk11-temurin ARG OBA_VERSION ENV OBA_VERSION=${OBA_VERSION} -RUN apt-get update && apt-get install -y jq unzip zip +RUN apt-get update && apt-get install -y --no-install-recommends jq unzip zip \ + && rm -rf /var/lib/apt/lists/* # Start configuring OBA WORKDIR /oba diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index 3fa9b68..c7314d4 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -144,6 +144,18 @@ download_bundle_inputs() { fi } +# xml_attr_escape STRING — escape a value for a double-quoted XML attribute. +# Manifest-derived ids/agency ids flow straight into bundle-context.xml, so a +# stray &, <, >, or " would otherwise produce malformed XML. +xml_attr_escape() { + local s="$1" + s="${s//&/&}" + s="${s///>}" + s="${s//\"/"}" + printf '%s' "$s" +} + # generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML # Bean id "gtfs-bundles" and bean name "entityReplacementStrategy" are looked up # by those exact names inside the federation builder — do not rename. @@ -161,15 +173,17 @@ generate_bundle_context_xml() { XMLHEAD - local feed_count i id agency + local feed_count i id agency path feed_count="$(jq -r '.feeds | length' "$manifest")" i=0 while [ "$i" -lt "$feed_count" ]; do id="$(jq -r ".feeds[$i].id" "$manifest")" agency="$(jq -r ".feeds[$i].defaultAgencyId" "$manifest")" + path="$(xml_attr_escape "${inputs_dir}/${id}.zip")" + agency="$(xml_attr_escape "$agency")" cat < - + XMLFEED @@ -257,7 +271,7 @@ run_single_mode() { if [[ -d "gtfs-out" ]]; then cd gtfs-out - zip ../gtfs_tidied.zip * + zip ../gtfs_tidied.zip ./* cd .. GTFS_ZIP_FILENAME="gtfs_tidied.zip" fi diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..7afbc0b --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,42 @@ +# CI-only override for the "Services with Bundler" job. Serves a small, +# committed GTFS fixture over the compose network so the job is deterministic +# and independent of the live Unitrans feed (see +# bin/testdata/services/make_services_fixture.sh for why). +# +# Compose merges `environment` and `depends_on` across -f files, so this file +# only carries the deltas; GTFS_URL here overrides the live default in +# docker-compose.yml for both the bundler and the app. The bundler/app wait on +# the fixture server's healthcheck so their wget can't race its startup. +# +# Usage: docker compose -f docker-compose.yml -f docker-compose.ci.yml ... +services: + fixture_server: + image: python:3.12-slim + container_name: oba_fixture_server + command: python -m http.server 8000 --directory /fixtures + volumes: + - ./bin/testdata/services:/fixtures:ro + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://localhost:8000/services-gtfs.zip') + interval: 2s + timeout: 3s + retries: 15 + start_period: 2s + + oba_bundler: + depends_on: + fixture_server: + condition: service_healthy + environment: + - GTFS_URL=http://fixture_server:8000/services-gtfs.zip + + oba_app: + depends_on: + fixture_server: + condition: service_healthy + environment: + - GTFS_URL=http://fixture_server:8000/services-gtfs.zip diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index 3fa9b68..c7314d4 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -144,6 +144,18 @@ download_bundle_inputs() { fi } +# xml_attr_escape STRING — escape a value for a double-quoted XML attribute. +# Manifest-derived ids/agency ids flow straight into bundle-context.xml, so a +# stray &, <, >, or " would otherwise produce malformed XML. +xml_attr_escape() { + local s="$1" + s="${s//&/&}" + s="${s///>}" + s="${s//\"/"}" + printf '%s' "$s" +} + # generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML # Bean id "gtfs-bundles" and bean name "entityReplacementStrategy" are looked up # by those exact names inside the federation builder — do not rename. @@ -161,15 +173,17 @@ generate_bundle_context_xml() { XMLHEAD - local feed_count i id agency + local feed_count i id agency path feed_count="$(jq -r '.feeds | length' "$manifest")" i=0 while [ "$i" -lt "$feed_count" ]; do id="$(jq -r ".feeds[$i].id" "$manifest")" agency="$(jq -r ".feeds[$i].defaultAgencyId" "$manifest")" + path="$(xml_attr_escape "${inputs_dir}/${id}.zip")" + agency="$(xml_attr_escape "$agency")" cat < - + XMLFEED @@ -257,7 +271,7 @@ run_single_mode() { if [[ -d "gtfs-out" ]]; then cd gtfs-out - zip ../gtfs_tidied.zip * + zip ../gtfs_tidied.zip ./* cd .. GTFS_ZIP_FILENAME="gtfs_tidied.zip" fi From 320355f4f8d746b154ab260301ecd183b60a1626 Mon Sep 17 00:00:00 2001 From: Aaron Brethorst Date: Sun, 5 Jul 2026 23:32:46 -0700 Subject: [PATCH 13/13] Fix xml_attr_escape for bash 5.2 (& in replacement = matched text) The bash ${//} version produced ', or " would otherwise produce malformed XML. +# stray &, <, >, or " would otherwise produce malformed XML. Uses sed rather +# than bash ${//} substitution: bash 5.2+ treats a literal & in the replacement +# as the matched text, which would mangle </>/" on newer runners. xml_attr_escape() { - local s="$1" - s="${s//&/&}" - s="${s///>}" - s="${s//\"/"}" - printf '%s' "$s" + printf '%s' "$1" | sed -e 's/&/\&/g' -e 's//\>/g' -e 's/"/\"/g' } # generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index c7314d4..69a9815 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -146,14 +146,11 @@ download_bundle_inputs() { # xml_attr_escape STRING — escape a value for a double-quoted XML attribute. # Manifest-derived ids/agency ids flow straight into bundle-context.xml, so a -# stray &, <, >, or " would otherwise produce malformed XML. +# stray &, <, >, or " would otherwise produce malformed XML. Uses sed rather +# than bash ${//} substitution: bash 5.2+ treats a literal & in the replacement +# as the matched text, which would mangle </>/" on newer runners. xml_attr_escape() { - local s="$1" - s="${s//&/&}" - s="${s///>}" - s="${s//\"/"}" - printf '%s' "$s" + printf '%s' "$1" | sed -e 's/&/\&/g' -e 's//\>/g' -e 's/"/\"/g' } # generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML