diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6a321e7..096ee27 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,6 +25,47 @@ jobs: working-directory: oba/config/template_renderer run: go test ./... + bundle_script: + name: build_bundle.sh unit tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Guard - oba and bundler scripts identical + run: diff oba/build_bundle.sh bundler/build_bundle.sh + + - name: Run script tests + run: bash bin/build_bundle_test.sh + + bundle_inputs_integration: + name: Bundle inputs integration (real builder) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Build oba image + run: | + docker buildx build \ + --load \ + -t oba-image:latest \ + ./oba + + - name: Run integration test + timeout-minutes: 20 + run: | + chmod +x bin/bundle_inputs_integration_test.sh bin/testdata/build_bundle/make_fixtures.sh + bash bin/bundle_inputs_integration_test.sh + image: name: Build Docker Image runs-on: ubuntu-latest @@ -52,6 +93,11 @@ jobs: services: name: Services with Bundler runs-on: ubuntu-latest + # Build against a small committed GTFS fixture served on the compose network + # instead of the live Unitrans feed, so this job is deterministic. See + # docker-compose.ci.yml and bin/testdata/services/make_services_fixture.sh. + env: + COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/README.md b/README.md index 5f89bfd..1f6f6a5 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,8 @@ You can find the latest published Docker images on Docker Hub: * `JDBC_PASSWORD` - The password for your database. * GTFS (Optional, required only when using `oba_app` independently) * `GTFS_URL` - The URL to the GTFS feed you want to use. + * `BUNDLE_INPUTS_URL` - URL to a bundle-inputs.json manifest for multi-input mode; specifies per-feed zips, agency IDs, and optional stop consolidation mapping. Mutually exclusive with GTFS_URL, GTFS_ZIP_FILENAME, and STOP_CONSOLIDATION_URL. + * `STOP_CONSOLIDATION_URL` - (Single-zip mode only) URL to a stop-consolidation mapping file applied during bundle build. Mutually exclusive with BUNDLE_INPUTS_URL. * GTFS-RT Support (Optional) * `TZ` - The timezone for the server. Ensure that the server's timezone matches the timezone specified in your static GTFS `agency.txt` file. The timezone format is the IANA standard, and [a full list of timezones can be found on Wikipedia](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List). * `GTFS_RT_FEEDS` - Preferred for configuring one OR many GTFS-RT feeds. A JSON array of feed objects; each object may contain `tripUpdatesUrl`, `vehiclePositionsUrl`, `alertsUrl`, `refreshInterval`, `agencyIds` (array), `feedApiKey`, and `feedApiValue`. Example: `[{"tripUpdatesUrl":"https://a/trips","agencyIds":["unitrans"]},{"vehiclePositionsUrl":"https://b/veh","agencyIds":["kcm"]}]`. When set, it takes precedence over the single-feed variables below. diff --git a/bin/build_bundle_test.sh b/bin/build_bundle_test.sh new file mode 100755 index 0000000..fca1b23 --- /dev/null +++ b/bin/build_bundle_test.sh @@ -0,0 +1,380 @@ +#!/bin/bash + +# Unit tests for oba/build_bundle.sh. Runs the script as a subprocess +# (run_script; main DOES execute) and exercises functions with stub binaries +# on PATH. +# Style follows bin/e2e_api_key_test.sh (pass/fail counters). + +set -u + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +SUT="$REPO_ROOT/oba/build_bundle.sh" + +passed=0 +failed=0 + +pass() { echo "PASS: $1"; passed=$((passed + 1)); } +fail() { echo "FAIL: $1"; failed=$((failed + 1)); } + +# The harness deliberately runs WITHOUT errexit: failing invocations of the +# script under test are expected and asserted on. + +# Run the script as a subprocess with a controlled env. +# usage: run_script [VAR=value ...] +# stdout+stderr -> $RUN_OUTPUT, exit code -> $RUN_STATUS +run_script() { + local stub_dir="$1"; shift + local tmp_out + tmp_out="$(mktemp)" + local path_prefix="" + [ -n "$stub_dir" ] && path_prefix="$stub_dir:" + env -i PATH="${path_prefix}/usr/bin:/bin:/usr/sbin:/sbin" HOME="$HOME" "$@" bash "$SUT" >"$tmp_out" 2>&1 + RUN_STATUS=$? + RUN_OUTPUT="$(cat "$tmp_out")" + rm -f "$tmp_out" +} + +assert_contains() { + local haystack="$1" needle="$2" label="$3" + if printf '%s' "$haystack" | grep -qF -- "$needle"; then + pass "$label" + else + fail "$label — expected to find: $needle" + printf 'GOT:\n%s\n' "$haystack" + fi +} + +echo "=== build_bundle.sh unit tests ===" + +# --- Mode validation: existing single-mode errors preserved ----------------- + +run_script "" GTFS_URL=http://x/gtfs.zip GTFS_ZIP_FILENAME=local.zip +[ "$RUN_STATUS" -ne 0 ] && pass "both GTFS_URL and GTFS_ZIP_FILENAME exits nonzero" \ + || fail "both GTFS_URL and GTFS_ZIP_FILENAME exits nonzero" +assert_contains "$RUN_OUTPUT" "Both GTFS_URL and GTFS_ZIP_FILENAME are set" "both-set error message preserved" + +run_script "" +[ "$RUN_STATUS" -ne 0 ] && pass "no mode env exits nonzero" || fail "no mode env exits nonzero" +assert_contains "$RUN_OUTPUT" "Neither GTFS_URL nor GTFS_ZIP_FILENAME is set" "neither-set error message preserved" + +# --- Single mode happy path with stubbed binaries --------------------------- + +STUBS="$(mktemp -d)" +WORK="$(mktemp -d)" + +cat > "$STUBS/wget" <<'EOF' +#!/bin/bash +# stub wget: records argv; creates the -O target +out="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else shift; fi +done +echo "wget $out" >> "$STUB_LOG" +echo "fake-zip" > "$out" +EOF + +cat > "$STUBS/gtfstidy" <<'EOF' +#!/bin/bash +echo "gtfstidy $*" >> "$STUB_LOG" +EOF + +cat > "$STUBS/java" <<'EOF' +#!/bin/bash +echo "java $*" >> "$STUB_LOG" +echo "cwd $PWD" >> "$STUB_LOG" +EOF + +chmod +x "$STUBS/wget" "$STUBS/gtfstidy" "$STUBS/java" + +STUB_LOG="$WORK/stub.log" +: > "$STUB_LOG" + +run_script "$STUBS" STUB_LOG="$STUB_LOG" BUNDLE_DIR="$WORK" \ + GTFS_URL=http://example.test/gtfs.zip TDF_BUILDER_JAR=/fake/builder.jar OBA_VERSION=test +[ "$RUN_STATUS" -eq 0 ] && pass "single mode (stubbed) exits 0" || fail "single mode (stubbed) exits 0: $RUN_OUTPUT" +assert_contains "$(cat "$STUB_LOG")" "wget $WORK/gtfs_pristine.zip" "single mode downloads to gtfs_pristine.zip" +assert_contains "$(cat "$STUB_LOG")" "gtfstidy" "single mode still runs gtfstidy" +assert_contains "$(cat "$STUB_LOG")" "java -Xss4m -Xmx3g -jar /fake/builder.jar ./gtfs_pristine.zip ." "single mode builder argv unchanged" +assert_contains "$(cat "$STUB_LOG")" "cwd $WORK" "builder runs from bundle dir" + +# --- Failed download now aborts (set -e hardening) --------------------------- + +cat > "$STUBS/wget" <<'EOF' +#!/bin/bash +exit 8 +EOF +chmod +x "$STUBS/wget" + +run_script "$STUBS" STUB_LOG="$STUB_LOG" BUNDLE_DIR="$WORK" GTFS_URL=http://example.test/gtfs.zip +[ "$RUN_STATUS" -ne 0 ] && pass "failed GTFS download aborts the build" || fail "failed GTFS download aborts the build" +assert_contains "$RUN_OUTPUT" "ERROR:" "failed download prints an ERROR: line" + +rm -rf "$STUBS" "$WORK" + +# --- Multi-input mode selection ---------------------------------------------- + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json GTFS_URL=http://x/gtfs.zip +[ "$RUN_STATUS" -ne 0 ] && pass "BUNDLE_INPUTS_URL + GTFS_URL exits nonzero" || fail "BUNDLE_INPUTS_URL + GTFS_URL exits nonzero" +assert_contains "$RUN_OUTPUT" "BUNDLE_INPUTS_URL cannot be combined" "combined-mode error message" + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json GTFS_ZIP_FILENAME=local.zip +[ "$RUN_STATUS" -ne 0 ] && pass "BUNDLE_INPUTS_URL + GTFS_ZIP_FILENAME exits nonzero" || fail "BUNDLE_INPUTS_URL + GTFS_ZIP_FILENAME exits nonzero" + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json STOP_CONSOLIDATION_URL=http://x/map.txt +[ "$RUN_STATUS" -ne 0 ] && pass "BUNDLE_INPUTS_URL + STOP_CONSOLIDATION_URL exits nonzero" || fail "BUNDLE_INPUTS_URL + STOP_CONSOLIDATION_URL exits nonzero" +assert_contains "$RUN_OUTPUT" "comes from the manifest" "consolidation-env-in-multi-mode error message" + +run_script "" BUNDLE_INPUTS_URL=http://x/bundle-inputs.json +assert_contains "$RUN_OUTPUT" "Multi-input mode" "BUNDLE_INPUTS_URL alone selects multi mode" + +# --- download_bundle_inputs --------------------------------------------------- + +TESTDATA="$REPO_ROOT/bin/testdata/build_bundle" +STUBS2="$(mktemp -d)" +WORK2="$(mktemp -d)" +SERVE="$(mktemp -d)" + +cat > "$STUBS2/wget" <<'EOF' +#!/bin/bash +# stub wget: serves $SERVE/; 404s (exit 8) when absent +out="" url="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else url="$1"; shift; fi +done +src="$SERVE/$(basename "$url")" +if [ -f "$src" ]; then cp "$src" "$out"; else exit 8; fi +EOF +chmod +x "$STUBS2/wget" + +# helper: run a snippet in a subshell that sources the SUT with a controlled env +# usage: run_sourced [VAR=value ...] +run_sourced() { + local snippet="$1"; shift + local tmp_out; tmp_out="$(mktemp)" + env -i PATH="$STUBS2:/usr/bin:/bin:/usr/sbin:/sbin" HOME="$HOME" SERVE="$SERVE" "$@" \ + bash -c "source '$SUT'; $snippet" >"$tmp_out" 2>&1 + RUN_STATUS=$? + RUN_OUTPUT="$(cat "$tmp_out")" + rm -f "$tmp_out" +} + +cp "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" +echo "zipbytes-metro" > "$SERVE/metro.zip" +echo "zipbytes-pierce" > "$SERVE/pierce.zip" +echo "1_M1 3_P1" > "$SERVE/StopConsolidation.txt" + +run_sourced "download_bundle_inputs && echo MAPPING=\$MAPPING_PATH" \ + BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -eq 0 ] && pass "download_bundle_inputs succeeds" || fail "download_bundle_inputs succeeds: $RUN_OUTPUT" +[ -f "$WORK2/inputs/metro.zip" ] && pass "metro.zip downloaded" || fail "metro.zip downloaded" +[ -f "$WORK2/inputs/pierce.zip" ] && pass "pierce.zip downloaded" || fail "pierce.zip downloaded" +[ -f "$WORK2/StopConsolidation.txt" ] && pass "mapping downloaded to StopConsolidation.txt" || fail "mapping downloaded to StopConsolidation.txt" +assert_contains "$RUN_OUTPUT" "MAPPING=$WORK2/StopConsolidation.txt" "MAPPING_PATH set" + +# no-mapping manifest → MAPPING_PATH empty +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +cp "$TESTDATA/bundle-inputs-no-mapping.json" "$SERVE/bundle-inputs.json" +run_sourced "download_bundle_inputs && echo MAPPING=[\$MAPPING_PATH]" \ + BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +assert_contains "$RUN_OUTPUT" "MAPPING=[]" "no stopConsolidationUrl -> empty MAPPING_PATH" +[ ! -e "$WORK2/StopConsolidation.txt" ] && pass "no mapping file created" || fail "no mapping file created" + +# unsupported version +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +echo '{"version": 2, "feeds": []}' > "$SERVE/bundle-inputs.json" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "unsupported manifest version fails" || fail "unsupported manifest version fails" +assert_contains "$RUN_OUTPUT" "ERROR: unsupported bundle-inputs version" "version error message" + +# empty feeds +echo '{"version": 1, "feeds": []}' > "$SERVE/bundle-inputs.json" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "empty feeds fails" || fail "empty feeds fails" +assert_contains "$RUN_OUTPUT" "ERROR: bundle-inputs manifest lists no feeds" "empty-feeds error message" + +# missing feed zip → ERROR naming the feed +cp "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" +rm -f "$SERVE/pierce.zip" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "missing feed download fails" || fail "missing feed download fails" +assert_contains "$RUN_OUTPUT" "ERROR: failed to download feed 'pierce'" "feed download error names the feed" +echo "zipbytes-pierce" > "$SERVE/pierce.zip" + +# sha256 mismatch → ERROR naming the feed +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +python3 - "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" <<'EOF' +import json, sys +m = json.load(open(sys.argv[1])) +m["feeds"][0]["sha256"] = "0" * 64 +json.dump(m, open(sys.argv[2], "w")) +EOF +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -ne 0 ] && pass "sha256 mismatch fails" || fail "sha256 mismatch fails" +assert_contains "$RUN_OUTPUT" "ERROR: sha256 mismatch for feed 'metro'" "sha mismatch error names the feed" + +# sha256 match succeeds +GOOD_SHA="$(sha256sum "$SERVE/metro.zip" | cut -d' ' -f1)" +python3 - "$TESTDATA/bundle-inputs.json" "$SERVE/bundle-inputs.json" "$GOOD_SHA" <<'EOF' +import json, sys +m = json.load(open(sys.argv[1])) +m["feeds"][0]["sha256"] = sys.argv[3] +json.dump(m, open(sys.argv[2], "w")) +EOF +rm -rf "$WORK2"; WORK2="$(mktemp -d)" +run_sourced "download_bundle_inputs" BUNDLE_DIR="$WORK2" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json +[ "$RUN_STATUS" -eq 0 ] && pass "matching sha256 passes" || fail "matching sha256 passes: $RUN_OUTPUT" + +rm -rf "$WORK2" +# NOTE: $STUBS2 and $SERVE are intentionally NOT removed here — Task 4's tests +# reuse run_sourced (which references $STUBS2 in PATH). Cleanup happens at the +# end of the file once all sourced-function tests are done. + +# --- generate_bundle_context_xml ---------------------------------------------- + +WORK3="$(mktemp -d)" +mkdir -p "$WORK3/inputs" +cp "$TESTDATA/bundle-inputs.json" "$WORK3/inputs/bundle-inputs.json" + +run_sourced "generate_bundle_context_xml '$WORK3/inputs/bundle-inputs.json' '$WORK3/inputs' '$WORK3/StopConsolidation.txt' '$WORK3/bundle-context.xml'" \ + BUNDLE_DIR="$WORK3" +[ "$RUN_STATUS" -eq 0 ] && pass "xml generation (with mapping) succeeds" || fail "xml generation (with mapping) succeeds: $RUN_OUTPUT" + +sed -e "s|@INPUTS@|$WORK3/inputs|g" -e "s|@MAPPING@|$WORK3/StopConsolidation.txt|g" \ + "$TESTDATA/golden-context-with-mapping.xml" > "$WORK3/expected.xml" +if diff -u "$WORK3/expected.xml" "$WORK3/bundle-context.xml"; then + pass "context XML matches golden (with mapping)" +else + fail "context XML matches golden (with mapping)" +fi + +run_sourced "generate_bundle_context_xml '$WORK3/inputs/bundle-inputs.json' '$WORK3/inputs' '' '$WORK3/bundle-context-nm.xml'" \ + BUNDLE_DIR="$WORK3" +sed -e "s|@INPUTS@|$WORK3/inputs|g" "$TESTDATA/golden-context-no-mapping.xml" > "$WORK3/expected-nm.xml" +if diff -u "$WORK3/expected-nm.xml" "$WORK3/bundle-context-nm.xml"; then + pass "context XML matches golden (no mapping)" +else + fail "context XML matches golden (no mapping)" +fi + +# XML metacharacters in manifest-derived ids/agency ids must be escaped, not +# emitted raw (which would produce malformed bundle-context.xml). +cat > "$WORK3/inputs/special.json" <<'EOF' +{"version": 1, "feeds": [{"id": "metro", "name": "n", "defaultAgencyId": "A&B "$STUBS3/wget" <<'EOF' +#!/bin/bash +out="" url="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else url="$1"; shift; fi +done +src="$SERVE/$(basename "$url")" +if [ -f "$src" ]; then cp "$src" "$out"; else exit 8; fi +EOF +cat > "$STUBS3/java" <<'EOF' +#!/bin/bash +echo "java $*" >> "$STUB_LOG" +echo "cwd $PWD" >> "$STUB_LOG" +EOF +cat > "$STUBS3/gtfstidy" <<'EOF' +#!/bin/bash +echo "gtfstidy WAS CALLED" >> "$STUB_LOG" +EOF +chmod +x "$STUBS3/wget" "$STUBS3/java" "$STUBS3/gtfstidy" + +cp "$TESTDATA/bundle-inputs.json" "$SERVE2/bundle-inputs.json" +echo "zipbytes-metro" > "$SERVE2/metro.zip" +echo "zipbytes-pierce" > "$SERVE2/pierce.zip" +echo "1_M1 3_P1" > "$SERVE2/StopConsolidation.txt" + +STUB_LOG="$WORK4/stub.log"; : > "$STUB_LOG" + +env -i PATH="$STUBS3:/usr/bin:/bin" HOME="$HOME" SERVE="$SERVE2" STUB_LOG="$STUB_LOG" \ + BUNDLE_DIR="$WORK4" BUNDLE_INPUTS_URL=http://fixtures.test/bundle-inputs.json \ + TDF_BUILDER_JAR=/fake/builder.jar \ + bash "$SUT" > "$WORK4/run.out" 2>&1 +MULTI_STATUS=$? +MULTI_OUTPUT="$(cat "$WORK4/run.out")" + +[ "$MULTI_STATUS" -eq 0 ] && pass "multi mode (stubbed) exits 0" || fail "multi mode (stubbed) exits 0: $MULTI_OUTPUT" +assert_contains "$(cat "$STUB_LOG")" "java -Xss4m -Xmx3g -jar /fake/builder.jar bundle-context.xml ." "multi mode builder argv" +assert_contains "$(cat "$STUB_LOG")" "cwd $WORK4" "multi mode builder cwd is bundle dir" +if grep -q "gtfstidy WAS CALLED" "$STUB_LOG"; then + fail "multi mode must not run gtfstidy" +else + pass "multi mode must not run gtfstidy" +fi +[ -f "$WORK4/bundle-context.xml" ] && pass "bundle-context.xml written to bundle dir" || fail "bundle-context.xml written to bundle dir" +[ -f "$WORK4/StopConsolidation.txt" ] && pass "StopConsolidation.txt present in bundle output dir" || fail "StopConsolidation.txt present in bundle output dir" +assert_contains "$MULTI_OUTPUT" "Multi-input mode" "multi mode banner printed" + +rm -rf "$STUBS3" "$WORK4" "$SERVE2" + +# --- single mode + STOP_CONSOLIDATION_URL -------------------------------------- + +STUBS4="$(mktemp -d)" +WORK5="$(mktemp -d)" +SERVE3="$(mktemp -d)" + +cat > "$STUBS4/wget" <<'EOF' +#!/bin/bash +out="" url="" +while [ $# -gt 0 ]; do + if [ "$1" = "-O" ]; then out="$2"; shift 2; else url="$1"; shift; fi +done +src="$SERVE/$(basename "$url")" +if [ -f "$src" ]; then cp "$src" "$out"; else exit 8; fi +EOF +cat > "$STUBS4/gtfstidy" <<'EOF' +#!/bin/bash +exit 0 +EOF +cat > "$STUBS4/java" <<'EOF' +#!/bin/bash +echo "java $*" >> "$STUB_LOG" +EOF +chmod +x "$STUBS4/wget" "$STUBS4/gtfstidy" "$STUBS4/java" + +echo "fake-gtfs" > "$SERVE3/gtfs.zip" +echo "1_M1 3_P1" > "$SERVE3/StopConsolidation.txt" +STUB_LOG="$WORK5/stub.log"; : > "$STUB_LOG" + +env -i PATH="$STUBS4:/usr/bin:/bin" HOME="$HOME" SERVE="$SERVE3" STUB_LOG="$STUB_LOG" \ + BUNDLE_DIR="$WORK5" GTFS_URL=http://fixtures.test/gtfs.zip \ + STOP_CONSOLIDATION_URL=http://fixtures.test/StopConsolidation.txt \ + TDF_BUILDER_JAR=/fake/builder.jar \ + bash "$SUT" > "$WORK5/run.out" 2>&1 +SC_STATUS=$? + +[ "$SC_STATUS" -eq 0 ] && pass "single mode + consolidation exits 0" || fail "single mode + consolidation exits 0: $(cat "$WORK5/run.out")" +[ -f "$WORK5/StopConsolidation.txt" ] && pass "single mode downloads mapping" || fail "single mode downloads mapping" +[ -f "$WORK5/consolidation-context.xml" ] && pass "single mode writes consolidation-context.xml" || fail "single mode writes consolidation-context.xml" +assert_contains "$(cat "$STUB_LOG")" "java -Xss4m -Xmx3g -jar /fake/builder.jar ./gtfs_pristine.zip consolidation-context.xml ." "single mode consolidation builder argv" +if grep -q "entityReplacementStrategy" "$WORK5/consolidation-context.xml" && ! grep -q "gtfs-bundles" "$WORK5/consolidation-context.xml"; then + pass "consolidation-context.xml has replacement beans only" +else + fail "consolidation-context.xml has replacement beans only" +fi + +rm -rf "$STUBS4" "$WORK5" "$SERVE3" +rm -rf "$STUBS2" "$SERVE" # deferred cleanup from the Task 3/4 sourced-function tests + +echo "" +echo "==============================" +echo "Results: $passed passed, $failed failed" +echo "==============================" +[ "$failed" -eq 0 ] diff --git a/bin/bundle_inputs_integration_test.sh b/bin/bundle_inputs_integration_test.sh new file mode 100755 index 0000000..a7e1eb0 --- /dev/null +++ b/bin/bundle_inputs_integration_test.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# Integration test: runs the REAL federation builder inside the built oba image +# in multi-input mode against fixture feeds. Requires: docker, python3, an +# `oba-image:latest` local image (build with: +# docker buildx build --load -t oba-image:latest ./oba ). +# Linux-first (uses --network host for the fixture HTTP server); on macOS run +# in CI instead. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +IMAGE="${OBA_IMAGE:-oba-image:latest}" +PORT=8123 + +passed=0 +failed=0 +pass() { echo "PASS: $1"; passed=$((passed + 1)); } +fail() { echo "FAIL: $1"; failed=$((failed + 1)); } + +SERVE="$(mktemp -d)" +OUT_VALID="$(mktemp -d)" +OUT_KM="$(mktemp -d)" + +bash "$REPO_ROOT/bin/testdata/build_bundle/make_fixtures.sh" "$SERVE" +cp "$REPO_ROOT/bin/testdata/build_bundle/mapping-valid.txt" "$SERVE/StopConsolidation.txt" + +cat > "$SERVE/bundle-inputs.json" </dev/null 2>&1 & +SERVER_PID=$! +trap 'kill $SERVER_PID 2>/dev/null; rm -rf "$SERVE" "$OUT_VALID" "$OUT_KM"' EXIT + +# Poll until the fixture server answers, rather than a fixed sleep that can race +# on a slow/loaded runner (the in-container wget calls would fail if it isn't up). +for _ in $(seq 1 40); do + curl -sf "http://127.0.0.1:${PORT}/bundle-inputs.json" >/dev/null 2>&1 && break + sleep 0.25 +done + +run_build() { + # --user matches the host user so the mounted output dir stays deletable + # by the test's cleanup trap (files written as root would survive rm -rf + # on CI runners). + local out_dir="$1" + docker run --rm --network host \ + --user "$(id -u):$(id -g)" \ + -v "$out_dir":/bundle \ + -e BUNDLE_INPUTS_URL="http://127.0.0.1:${PORT}/bundle-inputs.json" \ + --entrypoint bash \ + "$IMAGE" /oba/build_bundle.sh +} + +echo "--- Valid mapping build ---" +if run_build "$OUT_VALID" > "$OUT_VALID/build.log" 2>&1; then + pass "multi-input build with mapping exits 0" +else + fail "multi-input build with mapping exits 0" + tail -50 "$OUT_VALID/build.log" +fi + +[ -f "$OUT_VALID/TransitGraph.obj" ] && pass "bundle artifact TransitGraph.obj built" || fail "bundle artifact TransitGraph.obj built" +[ -f "$OUT_VALID/StopConsolidation.txt" ] && pass "StopConsolidation.txt in bundle output" || fail "StopConsolidation.txt in bundle output" +grep -q "gtfs=/bundle/inputs/metro.zip" "$OUT_VALID/build.log" && pass "metro loaded as its own bundle" || fail "metro loaded as its own bundle" +grep -q "gtfs=/bundle/inputs/pierce.zip" "$OUT_VALID/build.log" && pass "pierce loaded as its own bundle" || fail "pierce loaded as its own bundle" + +# Replaced stop is gone, keeper survives. TransitGraph.obj is one continuous +# Java serialization stream (TC_STRING tag + 2-byte length prefix + bytes, +# no line terminator), so `strings` glues "M2" to whatever stream token +# follows (observed: "M2pq") and an anchored `^M2$` line match is flaky — +# it happened to pass for the absence check and fail for the presence +# check on the same file. A direct byte-level substring grep on the raw +# file is reliable here because M2/P2 are short, unique 2-character ids +# that cannot occur elsewhere in the bundle's ASCII content (confirmed via +# `grep -aob "M2" TransitGraph.obj`: the sole hit is immediately preceded by +# the TC_STRING tag and a length=2 prefix, i.e. it is the serialized stop id +# field, not a coincidental substring). +grep -qa "P2" "$OUT_VALID/TransitGraph.obj" \ + && fail "replaced stop P2 absent from TransitGraph.obj" \ + || pass "replaced stop P2 absent from TransitGraph.obj" +grep -qa "M2" "$OUT_VALID/TransitGraph.obj" && pass "keeper M2 present in TransitGraph.obj" || fail "keeper M2 present in TransitGraph.obj" + +echo "--- Keeper-missing mapping build (behavior pin) ---" +cp "$REPO_ROOT/bin/testdata/build_bundle/mapping-keeper-missing.txt" "$SERVE/StopConsolidation.txt" +set +e +run_build "$OUT_KM" > "$OUT_KM/build.log" 2>&1 +KM_STATUS=$? +set -e +# Pin the observed behavior. "error replacing entity ... replacement not +# found" (hypothesized from reading app-modules source while writing this +# plan) does not appear against the real 2.7.1 builder jar — running this +# test is what surfaced that. What actually happens, verified by running +# it: the replaced stop (3_P2) is dropped from the graph exactly as in the +# valid-mapping build above, but nothing repoints the stopTime row that +# referenced it, so StopTimeEntriesFactory logs an ERROR for a stopTime +# with a null stop. That dangling reference — not a clean "replacement not +# found" warning — is the concrete harm that justifies obacloud excluding +# keeper-missing rows at publish. +grep -q "found stopTime without a stop id" "$OUT_KM/build.log" \ + && pass "keeper-missing line produces a dangling stopTime error" \ + || fail "keeper-missing line produces a dangling stopTime error" +grep -qa "P2" "$OUT_KM/TransitGraph.obj" \ + && fail "keeper-missing replaced stop P2 still absent from TransitGraph.obj" \ + || pass "keeper-missing replaced stop P2 still absent from TransitGraph.obj" +echo "keeper-missing build exit status: $KM_STATUS (recorded, not asserted — obacloud excludes such rows at publish)" + +echo "" +echo "==============================" +echo "Results: $passed passed, $failed failed" +echo "==============================" +[ "$failed" -eq 0 ] diff --git a/bin/testdata/build_bundle/bundle-inputs-no-mapping.json b/bin/testdata/build_bundle/bundle-inputs-no-mapping.json new file mode 100644 index 0000000..e2e24ed --- /dev/null +++ b/bin/testdata/build_bundle/bundle-inputs-no-mapping.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "feeds": [ + {"id": "metro", "name": "King County Metro", "defaultAgencyId": "1", "url": "http://fixtures.test/metro.zip"}, + {"id": "pierce", "name": "Pierce Transit", "defaultAgencyId": "3", "url": "http://fixtures.test/pierce.zip"} + ] +} diff --git a/bin/testdata/build_bundle/bundle-inputs.json b/bin/testdata/build_bundle/bundle-inputs.json new file mode 100644 index 0000000..e14d05b --- /dev/null +++ b/bin/testdata/build_bundle/bundle-inputs.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "feeds": [ + {"id": "metro", "name": "King County Metro", "defaultAgencyId": "1", "url": "http://fixtures.test/metro.zip"}, + {"id": "pierce", "name": "Pierce Transit", "defaultAgencyId": "3", "url": "http://fixtures.test/pierce.zip"} + ], + "stopConsolidationUrl": "http://fixtures.test/StopConsolidation.txt" +} diff --git a/bin/testdata/build_bundle/golden-context-no-mapping.xml b/bin/testdata/build_bundle/golden-context-no-mapping.xml new file mode 100644 index 0000000..5094667 --- /dev/null +++ b/bin/testdata/build_bundle/golden-context-no-mapping.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + diff --git a/bin/testdata/build_bundle/golden-context-with-mapping.xml b/bin/testdata/build_bundle/golden-context-with-mapping.xml new file mode 100644 index 0000000..791481e --- /dev/null +++ b/bin/testdata/build_bundle/golden-context-with-mapping.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bin/testdata/build_bundle/make_fixtures.sh b/bin/testdata/build_bundle/make_fixtures.sh new file mode 100755 index 0000000..e92d3dc --- /dev/null +++ b/bin/testdata/build_bundle/make_fixtures.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Builds two 3-stop fixture GTFS zips (metro: agency 1, pierce: agency 3) +# into the directory given as $1. Stops M2/P2 sit at identical coordinates +# so consolidating "1_M2 3_P2" is geometrically sensible. +set -euo pipefail +OUT="${1:?usage: make_fixtures.sh OUT_DIR}" +mkdir -p "$OUT" + +make_feed() { + local dir="$1" agency_id="$2" agency_name="$3" prefix="$4" + mkdir -p "$dir" + cat > "$dir/agency.txt" < "$dir/stops.txt" < "$dir/routes.txt" < "$dir/calendar.txt" < "$dir/trips.txt" < "$dir/stop_times.txt" < "$WORK/agency.txt" <<'EOF' +agency_id,agency_name,agency_url,agency_timezone +oba-test,OBA Test Transit,https://example.org,America/Los_Angeles +EOF + +# Seven stops strung ~1 km apart along a NE line near Davis, CA. Wide spacing +# keeps each stop distinct in the geospatial index. +cat > "$WORK/stops.txt" <<'EOF' +stop_id,stop_name,stop_lat,stop_lon +S1,First & Main,38.540000,-121.740000 +S2,Second & Oak,38.548000,-121.730000 +S3,Third & Elm,38.556000,-121.720000 +S4,Fourth & Pine,38.564000,-121.710000 +S5,Fifth & Cedar,38.572000,-121.700000 +S6,Sixth & Birch,38.580000,-121.690000 +S7,Seventh & Ash,38.588000,-121.680000 +EOF + +cat > "$WORK/routes.txt" <<'EOF' +route_id,agency_id,route_short_name,route_long_name,route_type +R1,oba-test,1,Test Line,3 +EOF + +cat > "$WORK/calendar.txt" <<'EOF' +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +WEEK,1,1,1,1,1,1,1,20200101,20401231 +EOF + +cat > "$WORK/trips.txt" <<'EOF' +route_id,service_id,trip_id +R1,WEEK,T1 +EOF + +cat > "$WORK/stop_times.txt" <<'EOF' +trip_id,arrival_time,departure_time,stop_id,stop_sequence +T1,08:00:00,08:00:00,S1,1 +T1,08:05:00,08:05:00,S2,2 +T1,08:10:00,08:10:00,S3,3 +T1,08:15:00,08:15:00,S4,4 +T1,08:20:00,08:20:00,S5,5 +T1,08:25:00,08:25:00,S6,6 +T1,08:30:00,08:30:00,S7,7 +EOF + +(cd "$WORK" && zip -q "$OUT_DIR/services-gtfs.zip" ./*.txt) +echo "wrote $OUT_DIR/services-gtfs.zip" diff --git a/bin/testdata/services/services-gtfs.zip b/bin/testdata/services/services-gtfs.zip new file mode 100644 index 0000000..4a1de2e Binary files /dev/null and b/bin/testdata/services/services-gtfs.zip differ diff --git a/bundler/Dockerfile b/bundler/Dockerfile index 48dcc85..b294a03 100644 --- a/bundler/Dockerfile +++ b/bundler/Dockerfile @@ -42,7 +42,8 @@ FROM tomcat:8.5.100-jdk11-temurin ARG OBA_VERSION ENV OBA_VERSION=${OBA_VERSION} -RUN apt-get update && apt-get install -y unzip zip +RUN apt-get update && apt-get install -y --no-install-recommends jq unzip zip \ + && rm -rf /var/lib/apt/lists/* # Start configuring OBA WORKDIR /oba diff --git a/bundler/build_bundle.sh b/bundler/build_bundle.sh index 0cc77dc..69a9815 100755 --- a/bundler/build_bundle.sh +++ b/bundler/build_bundle.sh @@ -16,17 +16,14 @@ # limitations under the License. # -# Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both -if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." - exit 1 -fi - -if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." - exit 1 -fi +set -euo pipefail +# Normalize env so `set -u` can't trip on optional vars. +GTFS_URL=${GTFS_URL:-} +GTFS_ZIP_FILENAME=${GTFS_ZIP_FILENAME:-} +BUNDLE_INPUTS_URL=${BUNDLE_INPUTS_URL:-} +STOP_CONSOLIDATION_URL=${STOP_CONSOLIDATION_URL:-} +OBA_VERSION=${OBA_VERSION:-} TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation-builder-withAllDependencies.jar} # Run gtfstidy (https://github.com/patrickbr/gtfstidy) with the following options enabled by default: @@ -41,43 +38,296 @@ TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation- # -D: drop erroneous entries from feed GTFS_TIDY_ARGS=${GTFS_TIDY_ARGS:-OscRCSmeD} -# Set default filename if using GTFS_URL -if [ -n "$GTFS_URL" ]; then - GTFS_ZIP_FILENAME="gtfs_pristine.zip" -fi +# Overridable for tests; production always uses /bundle. +BUNDLE_DIR=${BUNDLE_DIR:-/bundle} -echo "OBA Bundle Builder Starting" -if [ -n "$GTFS_URL" ]; then - echo "GTFS_URL: $GTFS_URL" -else - echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" -fi -echo "OBA Version: $OBA_VERSION" -echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" -echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" - -cd /bundle - -# Download GTFS file if URL is provided, otherwise use local file -if [ -n "$GTFS_URL" ]; then - wget -O ${GTFS_ZIP_FILENAME} ${GTFS_URL} -else - # Check if the local file exists - if [ ! -f "$GTFS_ZIP_FILENAME" ]; then - echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" +validate_mode_env() { + if [ -n "$BUNDLE_INPUTS_URL" ] && { [ -n "$GTFS_URL" ] || [ -n "$GTFS_ZIP_FILENAME" ]; }; then + echo "Error: BUNDLE_INPUTS_URL cannot be combined with GTFS_URL or GTFS_ZIP_FILENAME. Please provide only one mode." exit 1 fi -fi -gtfstidy -${GTFS_TIDY_ARGS} ${GTFS_ZIP_FILENAME} + if [ -n "$BUNDLE_INPUTS_URL" ] && [ -n "$STOP_CONSOLIDATION_URL" ]; then + echo "Error: STOP_CONSOLIDATION_URL cannot be set in multi-input mode; the mapping comes from the manifest's stopConsolidationUrl." + exit 1 + fi -if [[ -d "gtfs-out" ]]; then - cd gtfs-out - zip ../gtfs_tidied.zip * - cd .. - GTFS_ZIP_FILENAME="gtfs_tidied.zip" -fi + if [ -n "$BUNDLE_INPUTS_URL" ]; then + return 0 + fi + + # Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both + if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." + exit 1 + fi + + if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." + exit 1 + fi +} + +bundle_mode() { + if [ -n "$BUNDLE_INPUTS_URL" ]; then + echo "multi" + else + echo "single" + fi +} + +# fetch_url URL DEST LABEL — download with a one-line diagnostic on failure. +fetch_url() { + local url="$1" dest="$2" label="$3" + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download ${label} from ${url}" >&2 + exit 1 + fi +} + +# Downloads manifest, feed zips, and optional mapping. Requires jq. +# Sets MAPPING_PATH to the downloaded mapping file path, or "" when absent. +MAPPING_PATH="" + +download_bundle_inputs() { + local inputs_dir="$BUNDLE_DIR/inputs" + local manifest="$inputs_dir/bundle-inputs.json" + mkdir -p "$inputs_dir" + + fetch_url "$BUNDLE_INPUTS_URL" "$manifest" "bundle-inputs manifest" + + local version + version="$(jq -r '.version' "$manifest")" + if [ "$version" != "1" ]; then + echo "ERROR: unsupported bundle-inputs version: ${version}" >&2 + exit 1 + fi + + local feed_count + feed_count="$(jq -r '.feeds | length' "$manifest")" + if [ "$feed_count" -eq 0 ]; then + echo "ERROR: bundle-inputs manifest lists no feeds" >&2 + exit 1 + fi + + local i id url sha dest + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + url="$(jq -r ".feeds[$i].url" "$manifest")" + sha="$(jq -r ".feeds[$i].sha256 // empty" "$manifest")" + dest="$inputs_dir/${id}.zip" + + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download feed '${id}' from ${url}" >&2 + exit 1 + fi + + if [ -n "$sha" ]; then + if ! echo "${sha} ${dest}" | sha256sum -c - > /dev/null 2>&1; then + echo "ERROR: sha256 mismatch for feed '${id}' (${dest})" >&2 + exit 1 + fi + fi + i=$((i + 1)) + done + + local mapping_url + mapping_url="$(jq -r '.stopConsolidationUrl // empty' "$manifest")" + if [ -n "$mapping_url" ]; then + # StopConsolidation.txt is the hardcoded filename ConsolidatedStopsServiceImpl + # reads from the bundle directory at runtime. + MAPPING_PATH="$BUNDLE_DIR/StopConsolidation.txt" + fetch_url "$mapping_url" "$MAPPING_PATH" "stop consolidation mapping" + else + MAPPING_PATH="" + fi +} + +# xml_attr_escape STRING — escape a value for a double-quoted XML attribute. +# Manifest-derived ids/agency ids flow straight into bundle-context.xml, so a +# stray &, <, >, or " would otherwise produce malformed XML. Uses sed rather +# than bash ${//} substitution: bash 5.2+ treats a literal & in the replacement +# as the matched text, which would mangle </>/" on newer runners. +xml_attr_escape() { + printf '%s' "$1" | sed -e 's/&/\&/g' -e 's//\>/g' -e 's/"/\"/g' +} -# The JAR must be executed from within the same directory -# as the bundle, or else some necessary files are not generated. -java -Xss4m -Xmx3g -jar $TDF_BUILDER_JAR ./${GTFS_ZIP_FILENAME} . +# generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML +# Bean id "gtfs-bundles" and bean name "entityReplacementStrategy" are looked up +# by those exact names inside the federation builder — do not rename. +generate_bundle_context_xml() { + local manifest="$1" inputs_dir="$2" mapping_path="$3" out_xml="$4" + + { + cat <<'XMLHEAD' + + + + + + +XMLHEAD + + local feed_count i id agency path + feed_count="$(jq -r '.feeds | length' "$manifest")" + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + agency="$(jq -r ".feeds[$i].defaultAgencyId" "$manifest")" + path="$(xml_attr_escape "${inputs_dir}/${id}.zip")" + agency="$(xml_attr_escape "$agency")" + cat < + + + +XMLFEED + i=$((i + 1)) + done + + cat <<'XMLMID' + + + +XMLMID + + if [ -n "$mapping_path" ]; then + cat < + + + + + + + +XMLMAP + fi + + cat <<'XMLTAIL' + + +XMLTAIL + } > "$out_xml" +} + +# generate_single_mode_context_xml MAPPING_PATH OUT_XML — replacement beans only. +generate_single_mode_context_xml() { + local mapping_path="$1" out_xml="$2" + cat > "$out_xml" < + + + + + + + + + + + + +XMLSC +} + +run_single_mode() { + # Set default filename if using GTFS_URL + if [ -n "$GTFS_URL" ]; then + GTFS_ZIP_FILENAME="gtfs_pristine.zip" + fi + + echo "OBA Bundle Builder Starting" + if [ -n "$GTFS_URL" ]; then + echo "GTFS_URL: $GTFS_URL" + else + echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" + fi + echo "OBA Version: $OBA_VERSION" + echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + + cd "$BUNDLE_DIR" + + # Download GTFS file if URL is provided, otherwise use local file + if [ -n "$GTFS_URL" ]; then + fetch_url "$GTFS_URL" "$BUNDLE_DIR/$GTFS_ZIP_FILENAME" "GTFS feed" + else + # Check if the local file exists + if [ ! -f "$GTFS_ZIP_FILENAME" ]; then + echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" + exit 1 + fi + fi + + gtfstidy -"${GTFS_TIDY_ARGS}" "${GTFS_ZIP_FILENAME}" + + if [[ -d "gtfs-out" ]]; then + cd gtfs-out + zip ../gtfs_tidied.zip ./* + cd .. + GTFS_ZIP_FILENAME="gtfs_tidied.zip" + fi + + local context_args=() + if [ -n "$STOP_CONSOLIDATION_URL" ]; then + fetch_url "$STOP_CONSOLIDATION_URL" "$BUNDLE_DIR/StopConsolidation.txt" "stop consolidation mapping" + generate_single_mode_context_xml "$BUNDLE_DIR/StopConsolidation.txt" "$BUNDLE_DIR/consolidation-context.xml" + context_args=(consolidation-context.xml) + fi + + # The JAR must be executed from within the same directory + # as the bundle, or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" ${context_args[@]+"${context_args[@]}"} . +} + +run_multi_mode() { + echo "OBA Bundle Builder Starting" + echo "Multi-input mode: BUNDLE_INPUTS_URL: $BUNDLE_INPUTS_URL" + echo "OBA Version: $OBA_VERSION" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + echo "gtfstidy: skipped in multi-input mode (parity with the legacy multi-feed build)" + + mkdir -p "$BUNDLE_DIR" + cd "$BUNDLE_DIR" + + download_bundle_inputs + + generate_bundle_context_xml \ + "$BUNDLE_DIR/inputs/bundle-inputs.json" \ + "$BUNDLE_DIR/inputs" \ + "$MAPPING_PATH" \ + "$BUNDLE_DIR/bundle-context.xml" + + # Any non-.zip, non-directory arg before the last is treated as a Spring + # context path by FederatedTransitDataBundleCreatorMain; "." is the output dir. + # The JAR must be executed from within the same directory as the bundle, + # or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" bundle-context.xml . + + # MAPPING_PATH already lives at $BUNDLE_DIR/StopConsolidation.txt — the + # hardcoded name ConsolidatedStopsServiceImpl reads at runtime. Assert it + # survived the build rather than trusting the download. + if [ -n "$MAPPING_PATH" ] && [ ! -f "$MAPPING_PATH" ]; then + echo "ERROR: stop consolidation mapping missing from bundle output dir after build" >&2 + exit 1 + fi +} + +main() { + validate_mode_env + if [ "$(bundle_mode)" = "multi" ]; then + run_multi_mode + else + run_single_mode + fi +} + +# Main guard: allow tests to `source` this file without executing. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..7afbc0b --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,42 @@ +# CI-only override for the "Services with Bundler" job. Serves a small, +# committed GTFS fixture over the compose network so the job is deterministic +# and independent of the live Unitrans feed (see +# bin/testdata/services/make_services_fixture.sh for why). +# +# Compose merges `environment` and `depends_on` across -f files, so this file +# only carries the deltas; GTFS_URL here overrides the live default in +# docker-compose.yml for both the bundler and the app. The bundler/app wait on +# the fixture server's healthcheck so their wget can't race its startup. +# +# Usage: docker compose -f docker-compose.yml -f docker-compose.ci.yml ... +services: + fixture_server: + image: python:3.12-slim + container_name: oba_fixture_server + command: python -m http.server 8000 --directory /fixtures + volumes: + - ./bin/testdata/services:/fixtures:ro + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://localhost:8000/services-gtfs.zip') + interval: 2s + timeout: 3s + retries: 15 + start_period: 2s + + oba_bundler: + depends_on: + fixture_server: + condition: service_healthy + environment: + - GTFS_URL=http://fixture_server:8000/services-gtfs.zip + + oba_app: + depends_on: + fixture_server: + condition: service_healthy + environment: + - GTFS_URL=http://fixture_server:8000/services-gtfs.zip diff --git a/oba/build_bundle.sh b/oba/build_bundle.sh index 0cc77dc..69a9815 100755 --- a/oba/build_bundle.sh +++ b/oba/build_bundle.sh @@ -16,17 +16,14 @@ # limitations under the License. # -# Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both -if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." - exit 1 -fi - -if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then - echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." - exit 1 -fi +set -euo pipefail +# Normalize env so `set -u` can't trip on optional vars. +GTFS_URL=${GTFS_URL:-} +GTFS_ZIP_FILENAME=${GTFS_ZIP_FILENAME:-} +BUNDLE_INPUTS_URL=${BUNDLE_INPUTS_URL:-} +STOP_CONSOLIDATION_URL=${STOP_CONSOLIDATION_URL:-} +OBA_VERSION=${OBA_VERSION:-} TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation-builder-withAllDependencies.jar} # Run gtfstidy (https://github.com/patrickbr/gtfstidy) with the following options enabled by default: @@ -41,43 +38,296 @@ TDF_BUILDER_JAR=${TDF_BUILDER_JAR:-/oba/libs/onebusaway-transit-data-federation- # -D: drop erroneous entries from feed GTFS_TIDY_ARGS=${GTFS_TIDY_ARGS:-OscRCSmeD} -# Set default filename if using GTFS_URL -if [ -n "$GTFS_URL" ]; then - GTFS_ZIP_FILENAME="gtfs_pristine.zip" -fi +# Overridable for tests; production always uses /bundle. +BUNDLE_DIR=${BUNDLE_DIR:-/bundle} -echo "OBA Bundle Builder Starting" -if [ -n "$GTFS_URL" ]; then - echo "GTFS_URL: $GTFS_URL" -else - echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" -fi -echo "OBA Version: $OBA_VERSION" -echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" -echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" - -cd /bundle - -# Download GTFS file if URL is provided, otherwise use local file -if [ -n "$GTFS_URL" ]; then - wget -O ${GTFS_ZIP_FILENAME} ${GTFS_URL} -else - # Check if the local file exists - if [ ! -f "$GTFS_ZIP_FILENAME" ]; then - echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" +validate_mode_env() { + if [ -n "$BUNDLE_INPUTS_URL" ] && { [ -n "$GTFS_URL" ] || [ -n "$GTFS_ZIP_FILENAME" ]; }; then + echo "Error: BUNDLE_INPUTS_URL cannot be combined with GTFS_URL or GTFS_ZIP_FILENAME. Please provide only one mode." exit 1 fi -fi -gtfstidy -${GTFS_TIDY_ARGS} ${GTFS_ZIP_FILENAME} + if [ -n "$BUNDLE_INPUTS_URL" ] && [ -n "$STOP_CONSOLIDATION_URL" ]; then + echo "Error: STOP_CONSOLIDATION_URL cannot be set in multi-input mode; the mapping comes from the manifest's stopConsolidationUrl." + exit 1 + fi -if [[ -d "gtfs-out" ]]; then - cd gtfs-out - zip ../gtfs_tidied.zip * - cd .. - GTFS_ZIP_FILENAME="gtfs_tidied.zip" -fi + if [ -n "$BUNDLE_INPUTS_URL" ]; then + return 0 + fi + + # Check that either GTFS_URL or GTFS_ZIP_FILENAME is set, but not both + if [ -n "$GTFS_URL" ] && [ -n "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Both GTFS_URL and GTFS_ZIP_FILENAME are set. Please provide only one." + exit 1 + fi + + if [ -z "$GTFS_URL" ] && [ -z "$GTFS_ZIP_FILENAME" ]; then + echo "Error: Neither GTFS_URL nor GTFS_ZIP_FILENAME is set. Please provide one." + exit 1 + fi +} + +bundle_mode() { + if [ -n "$BUNDLE_INPUTS_URL" ]; then + echo "multi" + else + echo "single" + fi +} + +# fetch_url URL DEST LABEL — download with a one-line diagnostic on failure. +fetch_url() { + local url="$1" dest="$2" label="$3" + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download ${label} from ${url}" >&2 + exit 1 + fi +} + +# Downloads manifest, feed zips, and optional mapping. Requires jq. +# Sets MAPPING_PATH to the downloaded mapping file path, or "" when absent. +MAPPING_PATH="" + +download_bundle_inputs() { + local inputs_dir="$BUNDLE_DIR/inputs" + local manifest="$inputs_dir/bundle-inputs.json" + mkdir -p "$inputs_dir" + + fetch_url "$BUNDLE_INPUTS_URL" "$manifest" "bundle-inputs manifest" + + local version + version="$(jq -r '.version' "$manifest")" + if [ "$version" != "1" ]; then + echo "ERROR: unsupported bundle-inputs version: ${version}" >&2 + exit 1 + fi + + local feed_count + feed_count="$(jq -r '.feeds | length' "$manifest")" + if [ "$feed_count" -eq 0 ]; then + echo "ERROR: bundle-inputs manifest lists no feeds" >&2 + exit 1 + fi + + local i id url sha dest + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + url="$(jq -r ".feeds[$i].url" "$manifest")" + sha="$(jq -r ".feeds[$i].sha256 // empty" "$manifest")" + dest="$inputs_dir/${id}.zip" + + if ! wget -O "$dest" "$url"; then + echo "ERROR: failed to download feed '${id}' from ${url}" >&2 + exit 1 + fi + + if [ -n "$sha" ]; then + if ! echo "${sha} ${dest}" | sha256sum -c - > /dev/null 2>&1; then + echo "ERROR: sha256 mismatch for feed '${id}' (${dest})" >&2 + exit 1 + fi + fi + i=$((i + 1)) + done + + local mapping_url + mapping_url="$(jq -r '.stopConsolidationUrl // empty' "$manifest")" + if [ -n "$mapping_url" ]; then + # StopConsolidation.txt is the hardcoded filename ConsolidatedStopsServiceImpl + # reads from the bundle directory at runtime. + MAPPING_PATH="$BUNDLE_DIR/StopConsolidation.txt" + fetch_url "$mapping_url" "$MAPPING_PATH" "stop consolidation mapping" + else + MAPPING_PATH="" + fi +} + +# xml_attr_escape STRING — escape a value for a double-quoted XML attribute. +# Manifest-derived ids/agency ids flow straight into bundle-context.xml, so a +# stray &, <, >, or " would otherwise produce malformed XML. Uses sed rather +# than bash ${//} substitution: bash 5.2+ treats a literal & in the replacement +# as the matched text, which would mangle </>/" on newer runners. +xml_attr_escape() { + printf '%s' "$1" | sed -e 's/&/\&/g' -e 's//\>/g' -e 's/"/\"/g' +} -# The JAR must be executed from within the same directory -# as the bundle, or else some necessary files are not generated. -java -Xss4m -Xmx3g -jar $TDF_BUILDER_JAR ./${GTFS_ZIP_FILENAME} . +# generate_bundle_context_xml MANIFEST_JSON INPUTS_DIR MAPPING_PATH OUT_XML +# Bean id "gtfs-bundles" and bean name "entityReplacementStrategy" are looked up +# by those exact names inside the federation builder — do not rename. +generate_bundle_context_xml() { + local manifest="$1" inputs_dir="$2" mapping_path="$3" out_xml="$4" + + { + cat <<'XMLHEAD' + + + + + + +XMLHEAD + + local feed_count i id agency path + feed_count="$(jq -r '.feeds | length' "$manifest")" + i=0 + while [ "$i" -lt "$feed_count" ]; do + id="$(jq -r ".feeds[$i].id" "$manifest")" + agency="$(jq -r ".feeds[$i].defaultAgencyId" "$manifest")" + path="$(xml_attr_escape "${inputs_dir}/${id}.zip")" + agency="$(xml_attr_escape "$agency")" + cat < + + + +XMLFEED + i=$((i + 1)) + done + + cat <<'XMLMID' + + + +XMLMID + + if [ -n "$mapping_path" ]; then + cat < + + + + + + + +XMLMAP + fi + + cat <<'XMLTAIL' + + +XMLTAIL + } > "$out_xml" +} + +# generate_single_mode_context_xml MAPPING_PATH OUT_XML — replacement beans only. +generate_single_mode_context_xml() { + local mapping_path="$1" out_xml="$2" + cat > "$out_xml" < + + + + + + + + + + + + +XMLSC +} + +run_single_mode() { + # Set default filename if using GTFS_URL + if [ -n "$GTFS_URL" ]; then + GTFS_ZIP_FILENAME="gtfs_pristine.zip" + fi + + echo "OBA Bundle Builder Starting" + if [ -n "$GTFS_URL" ]; then + echo "GTFS_URL: $GTFS_URL" + else + echo "GTFS_ZIP_FILENAME: $GTFS_ZIP_FILENAME" + fi + echo "OBA Version: $OBA_VERSION" + echo "GTFS Tidy Args: $GTFS_TIDY_ARGS" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + + cd "$BUNDLE_DIR" + + # Download GTFS file if URL is provided, otherwise use local file + if [ -n "$GTFS_URL" ]; then + fetch_url "$GTFS_URL" "$BUNDLE_DIR/$GTFS_ZIP_FILENAME" "GTFS feed" + else + # Check if the local file exists + if [ ! -f "$GTFS_ZIP_FILENAME" ]; then + echo "Error: GTFS file not found: $GTFS_ZIP_FILENAME" + exit 1 + fi + fi + + gtfstidy -"${GTFS_TIDY_ARGS}" "${GTFS_ZIP_FILENAME}" + + if [[ -d "gtfs-out" ]]; then + cd gtfs-out + zip ../gtfs_tidied.zip ./* + cd .. + GTFS_ZIP_FILENAME="gtfs_tidied.zip" + fi + + local context_args=() + if [ -n "$STOP_CONSOLIDATION_URL" ]; then + fetch_url "$STOP_CONSOLIDATION_URL" "$BUNDLE_DIR/StopConsolidation.txt" "stop consolidation mapping" + generate_single_mode_context_xml "$BUNDLE_DIR/StopConsolidation.txt" "$BUNDLE_DIR/consolidation-context.xml" + context_args=(consolidation-context.xml) + fi + + # The JAR must be executed from within the same directory + # as the bundle, or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" ./"${GTFS_ZIP_FILENAME}" ${context_args[@]+"${context_args[@]}"} . +} + +run_multi_mode() { + echo "OBA Bundle Builder Starting" + echo "Multi-input mode: BUNDLE_INPUTS_URL: $BUNDLE_INPUTS_URL" + echo "OBA Version: $OBA_VERSION" + echo "TDF_BUILDER_JAR: $TDF_BUILDER_JAR" + echo "gtfstidy: skipped in multi-input mode (parity with the legacy multi-feed build)" + + mkdir -p "$BUNDLE_DIR" + cd "$BUNDLE_DIR" + + download_bundle_inputs + + generate_bundle_context_xml \ + "$BUNDLE_DIR/inputs/bundle-inputs.json" \ + "$BUNDLE_DIR/inputs" \ + "$MAPPING_PATH" \ + "$BUNDLE_DIR/bundle-context.xml" + + # Any non-.zip, non-directory arg before the last is treated as a Spring + # context path by FederatedTransitDataBundleCreatorMain; "." is the output dir. + # The JAR must be executed from within the same directory as the bundle, + # or else some necessary files are not generated. + java -Xss4m -Xmx3g -jar "$TDF_BUILDER_JAR" bundle-context.xml . + + # MAPPING_PATH already lives at $BUNDLE_DIR/StopConsolidation.txt — the + # hardcoded name ConsolidatedStopsServiceImpl reads at runtime. Assert it + # survived the build rather than trusting the download. + if [ -n "$MAPPING_PATH" ] && [ ! -f "$MAPPING_PATH" ]; then + echo "ERROR: stop consolidation mapping missing from bundle output dir after build" >&2 + exit 1 + fi +} + +main() { + validate_mode_env + if [ "$(bundle_mode)" = "multi" ]; then + run_multi_mode + else + run_single_mode + fi +} + +# Main guard: allow tests to `source` this file without executing. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi diff --git a/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs b/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs index 0ca4708..ec13377 100644 --- a/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs +++ b/oba/config/onebusaway-transit-data-federation-webapp-data-sources.xml.hbs @@ -77,6 +77,22 @@ {{/if}} + {{#if this.agencyIds.length}} + + + + {{#each this.agencyIds}}{{#if @first}}{{/if}}{{/each}} + + + {{else if this.agencyId}} + + + + + + {{/if}} + {{#if this.feedApiKey}} diff --git a/oba/config/template_renderer/main_test.go b/oba/config/template_renderer/main_test.go index ceba2dc..c702b44 100644 --- a/oba/config/template_renderer/main_test.go +++ b/oba/config/template_renderer/main_test.go @@ -217,3 +217,49 @@ func TestFederationTemplateSingularAgencyId(t *testing.T) { t.Errorf("did not expect plural agencyIds list:\n%s", out) } } + +func TestFederationTemplateStopModificationStrategy(t *testing.T) { + json := `{"FEEDS":[` + + `{"tripUpdatesUrl":"https://a/trips","agencyIds":["1","40"]},` + + `{"vehiclePositionsUrl":"https://b/vehicles","agencyId":"3"},` + + `{"alertsUrl":"https://c/alerts"}` + + `]}` + + out, err := renderTemplate(federationTemplatePath, json) + if err != nil { + t.Fatalf("renderTemplate returned an error: %v", err) + } + // Feeds with an agency id (list or singular) get the strategy; the + // agency-less feed must not. + if c := strings.Count(out, "ConsolidatedStopsModificationStrategy"); c != 2 { + t.Errorf("expected 2 strategy beans, got %d\n%s", c, out) + } + if c := strings.Count(out, ``); c != 2 { + t.Errorf("expected 2 stopModificationStrategy properties, got %d\n%s", c, out) + } + // Multi-agency feed: the strategy is namespaced to the FIRST listed agency. + // (Feed 1 uses the agencyIds list form, so the only `agencyId` property it + // renders is the strategy's.) + if !strings.Contains(out, ``) { + t.Errorf("strategy for multi-agency feed should use first agency:\n%s", out) + } + if strings.Contains(out, ``) { + t.Errorf("strategy must not be generated for non-first agencies:\n%s", out) + } +} + +func TestFederationTemplateStopModificationStrategySingularAgency(t *testing.T) { + json := `{"FEEDS":[{"tripUpdatesUrl":"https://x/trips","agencyId":"unitrans"}]}` + out, err := renderTemplate(federationTemplatePath, json) + if err != nil { + t.Fatalf("renderTemplate returned an error: %v", err) + } + if c := strings.Count(out, "ConsolidatedStopsModificationStrategy"); c != 1 { + t.Errorf("expected 1 strategy bean, got %d\n%s", c, out) + } + // Singular form renders agencyId twice: once on the GtfsRealtimeSource, + // once on the strategy bean. + if c := strings.Count(out, ``); c != 2 { + t.Errorf("expected agencyId on source and strategy, got %d\n%s", c, out) + } +}