From 183aacb7dcf9335255837bd7575797b5abdb5769 Mon Sep 17 00:00:00 2001 From: Sherley-Sonali Date: Tue, 23 Jun 2026 15:16:58 +0530 Subject: [PATCH 1/2] add GHA routing regression pipeline skeleton --- .github/actions/build-tiles/action.yml | 43 ++++++++ .github/actions/build-valhalla/action.yml | 68 +++++++++++++ .github/workflows/routing-regression.yml | 109 ++++++++++++++++++++ architecture.md | 118 ++++++++++++++++++++++ scripts/diff_responses.py | 68 +++++++++++++ scripts/run_routes.py | 43 ++++++++ 6 files changed, 449 insertions(+) create mode 100644 .github/actions/build-tiles/action.yml create mode 100644 .github/actions/build-valhalla/action.yml create mode 100644 .github/workflows/routing-regression.yml create mode 100644 architecture.md create mode 100644 scripts/diff_responses.py create mode 100644 scripts/run_routes.py diff --git a/.github/actions/build-tiles/action.yml b/.github/actions/build-tiles/action.yml new file mode 100644 index 0000000..9376a2f --- /dev/null +++ b/.github/actions/build-tiles/action.yml @@ -0,0 +1,43 @@ +name: Build Tiles +description: > + Builds a routable Valhalla graph extract from a PBF, using the Valhalla + install already present on the runner. Outputs a single tile_extract tar. + +inputs: + pbf_path: + description: "Path to the OSM PBF to build the graph from" + required: true + tile_dir: + description: "Working directory to build raw tiles into before bundling" + required: false + default: "valhalla_tiles" + +outputs: + tile_extract: + description: "Absolute path to the built, uncompressed tile extract tar" + value: ${{ steps.extract.outputs.tile_extract }} + +runs: + using: composite + steps: + - name: Build config, admins, and tiles + shell: bash + run: | + mkdir -p "${{ inputs.tile_dir }}" + valhalla_build_config \ + --mjolnir-tile-dir "${{ inputs.tile_dir }}" \ + --mjolnir-admin "${{ inputs.tile_dir }}/admins.sqlite" \ + --mjolnir-tile-extract "${{ inputs.tile_dir }}/tiles.tar" \ + > /tmp/valhalla-build.json + valhalla_build_admins -c /tmp/valhalla-build.json "${{ inputs.pbf_path }}" + valhalla_build_tiles -c /tmp/valhalla-build.json "${{ inputs.pbf_path }}" + + - name: Bundle tiles into a single extract + id: extract + shell: bash + run: | + valhalla_build_extract -c /tmp/valhalla-build.json -v + tile_count=$(find "${{ inputs.tile_dir }}" -name "*.gph" | wc -l) + echo "built $tile_count tile(s)" + test -f "${{ inputs.tile_dir }}/tiles.tar" + echo "tile_extract=$(pwd)/${{ inputs.tile_dir }}/tiles.tar" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/build-valhalla/action.yml b/.github/actions/build-valhalla/action.yml new file mode 100644 index 0000000..c77ea53 --- /dev/null +++ b/.github/actions/build-valhalla/action.yml @@ -0,0 +1,68 @@ +name: Build Valhalla +description: > + Builds Valhalla from source as a Python wheel and installs it into the + current job's environment. + +inputs: + valhalla_ref: + description: "Valhalla git ref to build (branch, tag, or commit SHA)" + required: true + python_version: + description: "Python version to build and install the wheel against" + required: false + default: "3.12" + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python_version }} + + - name: Sanitize ref for cache key + id: sanitize + shell: bash + run: echo "ref_slug=$(echo '${{ inputs.valhalla_ref }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" + + - name: Checkout Valhalla + uses: actions/checkout@v6 + with: + repository: valhalla/valhalla + ref: ${{ inputs.valhalla_ref }} + submodules: recursive + path: valhalla-src + + - name: Install system dependencies + shell: bash + run: bash valhalla-src/scripts/install-linux-deps.sh + + - name: Restore ccache + uses: actions/cache/restore@v4 + with: + path: ~/.cache/ccache + key: ccache-valhalla-${{ steps.sanitize.outputs.ref_slug }} + restore-keys: | + ccache-valhalla-master + + # pyproject.toml sets Release/ENABLE_PYTHON_BINDINGS/ENABLE_TESTS defaults + - name: Build wheel + shell: bash + run: | + cd valhalla-src + pip wheel . --wheel-dir /tmp/valhalla-dist \ + -Cbuild-dir=/tmp/valhalla-build \ + -Ccmake.define.ENABLE_TOOLS=ON \ + -Ccmake.define.ENABLE_SERVICES=OFF \ + -Ccmake.define.ENABLE_CCACHE=ON + + - name: Save ccache + if: always() + uses: actions/cache/save@v4 + with: + path: ~/.cache/ccache + key: ccache-valhalla-${{ steps.sanitize.outputs.ref_slug }} + + - name: Install wheel + shell: bash + run: pip install /tmp/valhalla-dist/*.whl diff --git a/.github/workflows/routing-regression.yml b/.github/workflows/routing-regression.yml new file mode 100644 index 0000000..cda68b7 --- /dev/null +++ b/.github/workflows/routing-regression.yml @@ -0,0 +1,109 @@ +name: Routing Regression + +on: + workflow_dispatch: + inputs: + valhalla_ref_old: + description: "Baseline Valhalla ref (the 'before')" + default: master + required: true + valhalla_ref_new: + description: "New Valhalla ref to test (the 'after' - branch, tag, or SHA)" + required: true + +# queue concurrent runs rather than cancel +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + regression: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - label: old + ref: ${{ inputs.valhalla_ref_old }} + - label: new + ref: ${{ inputs.valhalla_ref_new }} + + steps: + - name: Checkout RAD + uses: actions/checkout@v6 + with: + lfs: true + + - name: Checkout RAD-data + uses: actions/checkout@v6 + with: + repository: valhalla/RAD-data + path: RAD-data + + - name: Build Valhalla at ${{ matrix.label }} ref + uses: ./.github/actions/build-valhalla + with: + valhalla_ref: ${{ matrix.ref }} + python_version: "3.12" + + - name: Build tiles for ${{ matrix.label }} ref + id: tiles + uses: ./.github/actions/build-tiles + with: + pbf_path: data/liechtenstein_graph.osm.pbf + + - name: Run route requests + run: | + python3 scripts/run_routes.py \ + --tile-extract "${{ steps.tiles.outputs.tile_extract }}" \ + --requests RAD-data/requests/requests.jsonl \ + --output /tmp/responses-${{ matrix.label }}.jsonl + + # responses are only needed for the current run's diff-and-store job, 7 days is generous + - name: Upload responses artifact + uses: actions/upload-artifact@v4 + with: + name: responses-${{ matrix.label }} + path: /tmp/responses-${{ matrix.label }}.jsonl + retention-days: 7 + + diff-and-store: + runs-on: ubuntu-latest + needs: regression + permissions: + contents: write + + steps: + - name: Checkout RAD + uses: actions/checkout@v6 + + - name: Checkout RAD-data + uses: actions/checkout@v6 + with: + repository: valhalla/RAD-data + path: RAD-data + + - name: Download responses + uses: actions/download-artifact@v4 + with: + pattern: responses-* + path: /tmp/responses + merge-multiple: false + + # Debug step + - name: List downloaded artifacts + run: find /tmp/responses -type f + + - name: Compute diff and push to RAD-data + run: | + python3 scripts/diff_responses.py \ + --old /tmp/responses/responses-old/responses-old.jsonl \ + --new /tmp/responses/responses-new/responses-new.jsonl \ + --old-ref "${{ inputs.valhalla_ref_old }}" \ + --new-ref "${{ inputs.valhalla_ref_new }}" \ + --output RAD-data/diffs/${{ github.run_id }}.json + cd RAD-data + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add diffs/${{ github.run_id }}.json + git commit -m "regression: ${{ inputs.valhalla_ref_old }} vs ${{ inputs.valhalla_ref_new }} (run ${{ github.run_id }})" + git push https://x-access-token:${{ secrets.RAD_DATA_TOKEN }}@github.com/valhalla/RAD-data.git diff --git a/architecture.md b/architecture.md new file mode 100644 index 0000000..07ae4a5 --- /dev/null +++ b/architecture.md @@ -0,0 +1,118 @@ +# RAD — Routing Regression Pipeline + +```mermaid +flowchart TD + TRIGGER(["`**workflow_dispatch** + valhalla_ref_old · valhalla_ref_new`"]) + + TRIGGER -->|matrix: label=old| OLD_JOB + TRIGGER -->|matrix: label=new| NEW_JOB + + subgraph OLD_JOB ["old ref runner (ubuntu-latest)"] + direction TB + O1["① build-valhalla + composite action + ───────────────── + checkout valhalla at old ref + install-linux-deps.sh + restore ccache ← actions/cache + pip wheel . -Cbuild-dir=/tmp/valhalla-build + save ccache → actions/cache + pip install wheel"] + + O2["② build-tiles + composite action + ───────────────── + valhalla_build_config + valhalla_build_admins + valhalla_build_tiles + valhalla_build_extract → tiles.tar"] + + O3["③ run routes + ───────────────── + run_routes.py + pyvalhalla Actor in-process + → responses-old.jsonl"] + + O4[/"④ upload artifact + responses-old.jsonl"/] + + O1 --> O2 --> O3 --> O4 + end + + subgraph NEW_JOB ["new ref runner (ubuntu-latest)"] + direction TB + N1["① build-valhalla + composite action + ───────────────── + checkout valhalla at new ref + install-linux-deps.sh + restore ccache ← actions/cache + pip wheel . -Cbuild-dir=/tmp/valhalla-build + save ccache → actions/cache + pip install wheel"] + + N2["② build-tiles + composite action + ───────────────── + valhalla_build_config + valhalla_build_admins + valhalla_build_tiles + valhalla_build_extract → tiles.tar"] + + N3["③ run routes + ───────────────── + run_routes.py + pyvalhalla Actor in-process + → responses-new.jsonl"] + + N4[/"④ upload artifact + responses-new.jsonl"/] + + N1 --> N2 --> N3 --> N4 + end + + O4 --> DIFF + N4 --> DIFF + + subgraph DIFF ["diff-and-store (needs: regression)"] + direction TB + D1["download both artifacts + ───────────────── + responses-old.jsonl + responses-new.jsonl"] + + D2["diff_responses.py + ───────────────── + geometry delta · duration delta + instruction diff · severity score"] + + D3["git commit → valhalla/RAD-data + ───────────────── + diffs/{run_id}.json + push via RAD_DATA_TOKEN"] + + D1 --> D2 --> D3 + end + + CACHE[("actions/cache + ~/.cache/ccache + key: ccache-valhalla-{ref_slug}")] + + RADDATA[("valhalla/RAD-data + diffs/{run_id}.json")] + + O1 <-->|restore / save| CACHE + N1 <-->|restore / save| CACHE + D3 -->|git push| RADDATA +``` + +## Storage layout + +| Data | Where | Lifetime | +|------|--------|----------| +| ccache object files | GitHub `actions/cache` | until evicted (7-day LRU) | +| pyvalhalla wheel (16MB) | `/tmp/valhalla-dist` on runner | single job | +| graph tiles tar | `/tmp/valhalla_tiles` on runner | single job | +| route responses | GHA artifact | 7 days | +| diff results | `valhalla/RAD-data` git commit | permanent | diff --git a/scripts/diff_responses.py b/scripts/diff_responses.py new file mode 100644 index 0000000..2fe63b2 --- /dev/null +++ b/scripts/diff_responses.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +""" +Usage: + python3 scripts/diff_responses.py \ + --old /tmp/responses/responses-old/responses-old.jsonl \ + --new /tmp/responses/responses-new/responses-new.jsonl \ + --old-ref master \ + --new-ref feature-branch \ + --output RAD-data/diffs/12345678.json +""" + +import argparse +import json +from datetime import UTC, datetime + + +def load_jsonl(path): + return [json.loads(line) for line in open(path)] + + +def diff_entry(old, new): + # TODO: expand with geometry comparison, maneuver diffs, etc. + old_dur = old["response"].get("trip", {}).get("summary", {}).get("time", None) + new_dur = new["response"].get("trip", {}).get("summary", {}).get("time", None) + return { + "request": old["request"], + "old_duration": old_dur, + "new_duration": new_dur, + "duration_delta_s": (new_dur - old_dur) if (old_dur and new_dur) else None, + "old_status": old["status"], + "new_status": new["status"], + } + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--old", required=True) + p.add_argument("--new", required=True) + p.add_argument("--old-ref", required=True) + p.add_argument("--new-ref", required=True) + p.add_argument("--output", required=True) + args = p.parse_args() + + old_rows = load_jsonl(args.old) + new_rows = load_jsonl(args.new) + + diffs = [diff_entry(o, n) for o, n in zip(old_rows, new_rows, strict=False)] + changed = [d for d in diffs if d["duration_delta_s"] not in (None, 0)] + + result = { + "meta": { + "old_ref": args.old_ref, + "new_ref": args.new_ref, + "generated_at": datetime.now(UTC).isoformat(), + "total_requests": len(diffs), + "changed_routes": len(changed), + }, + "diffs": diffs, + } + + with open(args.output, "w") as f: + json.dump(result, f, indent=2) + + print(f"Done: {len(changed)}/{len(diffs)} routes changed") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_routes.py b/scripts/run_routes.py new file mode 100644 index 0000000..7fb0a1c --- /dev/null +++ b/scripts/run_routes.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +""" +Usage: + python3 scripts/run_routes.py \ + --tile-extract /path/to/tiles.tar \ + --requests RAD-data/requests/requests.jsonl \ + --output /tmp/responses-old.jsonl +""" + +import argparse +import json +from pathlib import Path + +from valhalla import Actor, ValhallaError +from valhalla.config import _sanitize_config, default_config + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--tile-extract", required=True, help="Path to a built Valhalla tile extract (.tar)") + p.add_argument("--requests", required=True) + p.add_argument("--output", required=True) + args = p.parse_args() + + config = _sanitize_config(default_config.copy()) + config["mjolnir"]["tile_extract"] = str(Path(args.tile_extract).resolve()) + config["mjolnir"]["tile_dir"] = "" + actor = Actor(config) + + with open(args.requests) as req_f, open(args.output, "w") as out_f: + for line in req_f: + request = json.loads(line) + try: + response = actor.route(request) + status = "ok" + except ValhallaError as e: + response = {"error": str(e)} + status = "error" + out_f.write(json.dumps({"request": request, "response": response, "status": status}) + "\n") + + +if __name__ == "__main__": + main() From 6ad6774fe51ef7e3038102120e1586f344c779b2 Mon Sep 17 00:00:00 2001 From: Sherley-Sonali Date: Wed, 24 Jun 2026 15:27:21 +0530 Subject: [PATCH 2/2] address review: cmake flags, shell script, move scripts to debug/ --- .github/actions/build-tiles/action.yml | 21 ++-------------- .github/actions/build-valhalla/action.yml | 5 ++-- .github/workflows/routing-regression.yml | 6 +++-- .github/workflows/scripts/build_extract.sh | 29 ++++++++++++++++++++++ {scripts => debug}/diff_responses.py | 0 {scripts => debug}/run_routes.py | 0 6 files changed, 38 insertions(+), 23 deletions(-) create mode 100755 .github/workflows/scripts/build_extract.sh rename {scripts => debug}/diff_responses.py (100%) rename {scripts => debug}/run_routes.py (100%) diff --git a/.github/actions/build-tiles/action.yml b/.github/actions/build-tiles/action.yml index 9376a2f..130afab 100644 --- a/.github/actions/build-tiles/action.yml +++ b/.github/actions/build-tiles/action.yml @@ -20,24 +20,7 @@ outputs: runs: using: composite steps: - - name: Build config, admins, and tiles - shell: bash - run: | - mkdir -p "${{ inputs.tile_dir }}" - valhalla_build_config \ - --mjolnir-tile-dir "${{ inputs.tile_dir }}" \ - --mjolnir-admin "${{ inputs.tile_dir }}/admins.sqlite" \ - --mjolnir-tile-extract "${{ inputs.tile_dir }}/tiles.tar" \ - > /tmp/valhalla-build.json - valhalla_build_admins -c /tmp/valhalla-build.json "${{ inputs.pbf_path }}" - valhalla_build_tiles -c /tmp/valhalla-build.json "${{ inputs.pbf_path }}" - - - name: Bundle tiles into a single extract + - name: Build tiles and extract id: extract shell: bash - run: | - valhalla_build_extract -c /tmp/valhalla-build.json -v - tile_count=$(find "${{ inputs.tile_dir }}" -name "*.gph" | wc -l) - echo "built $tile_count tile(s)" - test -f "${{ inputs.tile_dir }}/tiles.tar" - echo "tile_extract=$(pwd)/${{ inputs.tile_dir }}/tiles.tar" >> "$GITHUB_OUTPUT" + run: .github/workflows/scripts/build_extract.sh "${{ inputs.pbf_path }}" "${{ inputs.tile_dir }}" diff --git a/.github/actions/build-valhalla/action.yml b/.github/actions/build-valhalla/action.yml index c77ea53..30a4f3f 100644 --- a/.github/actions/build-valhalla/action.yml +++ b/.github/actions/build-valhalla/action.yml @@ -52,9 +52,10 @@ runs: cd valhalla-src pip wheel . --wheel-dir /tmp/valhalla-dist \ -Cbuild-dir=/tmp/valhalla-build \ - -Ccmake.define.ENABLE_TOOLS=ON \ -Ccmake.define.ENABLE_SERVICES=OFF \ - -Ccmake.define.ENABLE_CCACHE=ON + -Ccmake.define.ENABLE_CCACHE=ON \ + -Ccmake.define.ENABLE_GEOTIFF=OFF \ + -Ccmake.define.ENABLE_LZ4=OFF - name: Save ccache if: always() diff --git a/.github/workflows/routing-regression.yml b/.github/workflows/routing-regression.yml index cda68b7..34b66b8 100644 --- a/.github/workflows/routing-regression.yml +++ b/.github/workflows/routing-regression.yml @@ -53,7 +53,8 @@ jobs: - name: Run route requests run: | - python3 scripts/run_routes.py \ + # TODO: WIP , Move to scripts/ once done. + python3 debug/run_routes.py \ --tile-extract "${{ steps.tiles.outputs.tile_extract }}" \ --requests RAD-data/requests/requests.jsonl \ --output /tmp/responses-${{ matrix.label }}.jsonl @@ -95,7 +96,8 @@ jobs: - name: Compute diff and push to RAD-data run: | - python3 scripts/diff_responses.py \ + # TODO: WIP , Move to scripts/ once done. + python3 debug/diff_responses.py \ --old /tmp/responses/responses-old/responses-old.jsonl \ --new /tmp/responses/responses-new/responses-new.jsonl \ --old-ref "${{ inputs.valhalla_ref_old }}" \ diff --git a/.github/workflows/scripts/build_extract.sh b/.github/workflows/scripts/build_extract.sh new file mode 100755 index 0000000..3c5d40b --- /dev/null +++ b/.github/workflows/scripts/build_extract.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Builds Valhalla graph tiles from a PBF and bundles them into a single +# tar extract. Outputs the absolute path via GITHUB_OUTPUT. +# +# Usage: build_extract.sh + +PBF_PATH="${1:?first argument must be the path to the OSM PBF file}" +TILE_DIR="${2:?second argument must be the tile output directory}" +CONFIG="/tmp/valhalla-build.json" + +mkdir -p "$TILE_DIR" + +valhalla_build_config \ + --mjolnir-tile-dir "$TILE_DIR" \ + --mjolnir-admin "$TILE_DIR/admins.sqlite" \ + --mjolnir-tile-extract "$TILE_DIR/tiles.tar" \ + > "$CONFIG" + +valhalla_build_admins -c "$CONFIG" "$PBF_PATH" +valhalla_build_tiles -c "$CONFIG" "$PBF_PATH" +valhalla_build_extract -c "$CONFIG" -v + +tile_count=$(find "$TILE_DIR" -name "*.gph" | wc -l) +echo "built ${tile_count} tile(s)" + +test -f "$TILE_DIR/tiles.tar" +echo "tile_extract=$(realpath "$TILE_DIR/tiles.tar")" >> "$GITHUB_OUTPUT" diff --git a/scripts/diff_responses.py b/debug/diff_responses.py similarity index 100% rename from scripts/diff_responses.py rename to debug/diff_responses.py diff --git a/scripts/run_routes.py b/debug/run_routes.py similarity index 100% rename from scripts/run_routes.py rename to debug/run_routes.py