diff --git a/.github/workflows/performance-analysis.yml b/.github/workflows/performance-analysis.yml new file mode 100644 index 000000000..67f5877e9 --- /dev/null +++ b/.github/workflows/performance-analysis.yml @@ -0,0 +1,170 @@ +name: Remote Read Performance Analysis + +permissions: + contents: read + +on: + workflow_dispatch: + +env: + BENCHMARK_S3_PATH: "https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a" + BENCHMARK_AWS_REGION: "us-east-2" + BENCHMARK_OBJECT_NAME: "ElectricalSeriesAp" + BENCHMARK_START_INDICES: "0,0" + BENCHMARK_COUNT_INDICES: "10,1" + BENCHMARK_REPETITIONS: "10" + +jobs: + benchmark: + name: HDF5 ${{ matrix.hdf5 }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + strategy: + fail-fast: false + matrix: + include: + - hdf5: "1.14" + conda_hdf5_spec: "hdf5=1.14" + hdf5_root: "$CONDA_PREFIX" + - hdf5: "2.2" + conda_hdf5_spec: "" + hdf5_root: "$RUNNER_TEMP/hdf5-install" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: aqnwb-performance + channels: conda-forge + conda-remove-defaults: true + + - name: Install benchmark dependencies + run: | + conda install -y \ + python=3.12 \ + cmake \ + ninja \ + numpy \ + libcurl \ + pip + if [ -n "${{ matrix.conda_hdf5_spec }}" ]; then + conda install -y "${{ matrix.conda_hdf5_spec }}" + fi + if [ "${{ matrix.hdf5 }}" = "2.2" ]; then + conda install -y aws-c-s3 + fi + + - name: Build HDF5 2.2 from source + if: matrix.hdf5 == '2.2' + run: | + git clone --depth 1 --branch hdf5_2_2_0 https://github.com/HDFGroup/hdf5.git hdf5-src + cmake -S hdf5-src -B hdf5-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$RUNNER_TEMP/hdf5-install" \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" \ + -DHDF5_ENABLE_ROS3_VFD=ON \ + -DHDF5_BUILD_CPP_LIB=ON \ + -DHDF5_BUILD_TOOLS=OFF \ + -DHDF5_BUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=ON + cmake --build hdf5-build --config Release -j 2 + cmake --install hdf5-build + + - name: Install Python benchmark stack against selected HDF5 + run: | + export HDF5_DIR="${{ matrix.hdf5_root }}" + export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" + python -m pip install --no-binary=h5py h5py + python -m pip install pynwb remfile + + - name: Configure and build aqnwb + run: | + export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DAQNWB_USE_REMFILE=ON \ + -DCMAKE_PREFIX_PATH="${{ matrix.hdf5_root }};$CONDA_PREFIX" \ + -DHDF5_ROOT="${{ matrix.hdf5_root }}" \ + -DHDF5_USE_STATIC_LIBRARIES=OFF + cmake --build build --config Release -j 2 + cmake --install build --prefix "$RUNNER_TEMP/aqnwb-install" + + - name: Build remote_read_benchmark demo + run: | + export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" + cmake -S demo/remote_read_benchmark -B demo/remote_read_benchmark/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$RUNNER_TEMP/aqnwb-install;${{ matrix.hdf5_root }};$CONDA_PREFIX" \ + -DHDF5_ROOT="${{ matrix.hdf5_root }}" \ + -DHDF5_USE_STATIC_LIBRARIES=OFF + cmake --build demo/remote_read_benchmark/build --config Release -j 2 + + - name: Run benchmark suite + run: | + export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" + python demo/remote_read_benchmark/run_benchmark_matrix.py \ + --cpp-binary demo/remote_read_benchmark/build/bin/remote_read_benchmark \ + --python-script demo/remote_read_benchmark/benchmark.py \ + --python-executable python \ + --hdf5-version "${{ matrix.hdf5 }}" \ + --repetitions "$BENCHMARK_REPETITIONS" \ + --output-json "artifacts/benchmark-results-${{ matrix.hdf5 }}.json" \ + "$BENCHMARK_S3_PATH" \ + "$BENCHMARK_AWS_REGION" \ + "$BENCHMARK_OBJECT_NAME" \ + "$BENCHMARK_START_INDICES" \ + "$BENCHMARK_COUNT_INDICES" + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-hdf5-${{ matrix.hdf5 }} + path: artifacts/benchmark-results-${{ matrix.hdf5 }}.json + + summarize: + name: Summarize benchmark results + needs: benchmark + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download benchmark artifacts + uses: actions/download-artifact@v4 + with: + path: benchmark-artifacts + pattern: benchmark-results-hdf5-* + merge-multiple: true + + - name: Generate markdown summary + run: | + python demo/remote_read_benchmark/summarize_benchmark_results.py \ + --results-dir benchmark-artifacts \ + --output-markdown benchmark-artifacts/benchmark-summary.md \ + --output-json benchmark-artifacts/benchmark-summary.json \ + --output-all-runs-csv benchmark-artifacts/benchmark-all-runs.csv \ + --output-fastest-runs-csv benchmark-artifacts/benchmark-fastest-runs.csv + cat benchmark-artifacts/benchmark-summary.md >> "$GITHUB_STEP_SUMMARY" + echo "===== Remote Read Benchmark Summary (Markdown) =====" + cat benchmark-artifacts/benchmark-summary.md + echo "===== All Runs CSV =====" + cat benchmark-artifacts/benchmark-all-runs.csv + echo "===== Fastest Runs CSV =====" + cat benchmark-artifacts/benchmark-fastest-runs.csv + + - name: Upload benchmark summary + uses: actions/upload-artifact@v4 + with: + name: benchmark-summary + path: | + benchmark-artifacts/benchmark-summary.md + benchmark-artifacts/benchmark-summary.json + benchmark-artifacts/benchmark-all-runs.csv + benchmark-artifacts/benchmark-fastest-runs.csv diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e738604..2e2731ad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Added `HDF5IO::openRemote()` method to read remote NWB files over HTTP(S) using the [remfile-cpp](https://github.com/catalystneuro/remfile-cpp) virtual file driver (a C++ port of the Python [remfile](https://github.com/magland/remfile) package), imported as an optional CMake dependency (`AQNWB_USE_REMFILE`, requires `libcurl`). Unlike ROS3, remfile does not require HDF5 to be built with ROS3 support and works with any HTTP(S) server that supports byte-range requests. (@bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) * Added demo for benchmarking ROS3 and remfile performance and comparing with PyNWB S3 reads (`demo/remote_read_benchmark`). (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) * Added tutorial on using the ROS3 and remfile drivers to read NWB files in S3 (`docs/pages/userdocs/reads3.dox`) (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309)) + * Added a dedicated `performance-analysis.yml` workflow plus benchmark helper scripts to run the remote-read benchmarks repeatedly across HDF5 1.14 and 2.2 and publish markdown summary tables. (@copilot) * Added `ElectricalSeries::writeAllChannels` method and `IO::writeElectricalSeriesData` overload to simplify zero-copy interleaved multichannel writes. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293)) * Added `ElectricalSeries::channelsAtSameSampleOffset` method to check if all channels are at the same sample offset, which is a requirement for using `writeAllChannels`. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293)) * Added new `BaseIO::findObject` and `RegisteredType::findOwnedObject` methods to simplify searching for objects by name. Added `HDF5IO::findObject` override method to optimize the search for HDF5 objects. (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308)) diff --git a/demo/remote_read_benchmark/README.md b/demo/remote_read_benchmark/README.md index 6989ed8c6..bd19a7024 100644 --- a/demo/remote_read_benchmark/README.md +++ b/demo/remote_read_benchmark/README.md @@ -115,8 +115,37 @@ By default, the script will attempt to use the ROS3 driver (falling back to `remfile` instead of the ROS3 driver (e.g. to compare the performance of the two read strategies), pass the `--force-remfile` flag. +For automation, both the C++ and Python benchmarks also support machine-readable +JSON output: + +```bash +./remote_read_benchmark \ + "https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a" \ + "us-east-2" \ + "ElectricalSeriesAp" \ + "0,0" \ + "10,1" \ + --json + +python demo/remote_read_benchmark/benchmark.py \ + "https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a" \ + "us-east-2" \ + "ElectricalSeriesAp" \ + "0,0" \ + "10,1" \ + --driver ros3 \ + --strict-driver \ + --output-format json +``` + +The helper scripts `run_benchmark_matrix.py` and `summarize_benchmark_results.py` +are used by the dedicated GitHub Actions performance workflow to run all +benchmark variants repeatedly and render markdown summary tables. + ## Code Structure - `main.cpp`: Contains the C++ benchmarking logic and timing measurements. - `CMakeLists.txt`: CMake configuration file for building the C++ project. - `benchmark.py`: Contains the Python benchmarking logic using PyNWB. +- `run_benchmark_matrix.py`: Repeats all benchmark variants for one HDF5 environment and writes raw JSON results. +- `summarize_benchmark_results.py`: Combines JSON artifacts into markdown summary tables. diff --git a/demo/remote_read_benchmark/benchmark.py b/demo/remote_read_benchmark/benchmark.py index a3ca82784..35fbe1dc8 100644 --- a/demo/remote_read_benchmark/benchmark.py +++ b/demo/remote_read_benchmark/benchmark.py @@ -25,41 +25,44 @@ """ import time +import json import argparse -from typing import List, Any +from typing import List, Any, Tuple, Dict from pynwb import NWBHDF5IO, NWBFile import remfile import h5py -def read_io(s3_path: str, aws_region: str, force_remfile: bool = False) -> NWBHDF5IO: +def read_io(s3_path: str, aws_region: str, driver: str, strict_driver: bool = False) -> Tuple[NWBHDF5IO, str]: """ - Opens the NWB file using NWBHDF5IO with ROS3 VFD. + Opens the NWB file using NWBHDF5IO with the requested driver. Equivalent to C++ read_io. :param s3_path: S3 URL of the NWB file. :param aws_region: AWS region (e.g., us-east-2). - :param force_remfile: If True, remfile is used directly instead of attempting - to use the ROS3 driver. This is useful for benchmarking/comparing the - two different read strategies, or on systems where h5py was not built - with ROS3 support. - :return: An instance of NWBHDF5IO. + :param driver: Requested driver ("ros3" or "remfile"). + :param strict_driver: If True, raise an error instead of falling back to remfile. + :return: A tuple of the NWBHDF5IO instance and the actual driver used. """ - def read_io_remfile(s3_path): - print("Using remfile to read the NWB file from S3.") - rem_file = remfile.File(s3_path) + def read_io_remfile(remote_path: str) -> NWBHDF5IO: + rem_file = remfile.File(remote_path) h5py_file = h5py.File(rem_file, "r") io = NWBHDF5IO(file=h5py_file) return io - if force_remfile: - return read_io_remfile(s3_path) + if driver == "remfile": + return read_io_remfile(s3_path), "remfile" + + if driver != "ros3": + raise ValueError(f"Unknown driver '{driver}' (expected 'ros3' or 'remfile')") # In PyNWB, NWBHDF5IO handles the HDF5 file opening and ROS3 configuration. try: - return NWBHDF5IO(s3_path, mode='r', driver='ros3', aws_region=aws_region) + return NWBHDF5IO(s3_path, mode='r', driver='ros3', aws_region=aws_region), "ros3" except (ImportError, ValueError) as e: + if strict_driver: + raise RuntimeError("h5py with ROS3 support is required for the requested ROS3 benchmark.") from e print("h5py with ROS3 support is required. Falling back to remfile.") - return read_io_remfile(s3_path) + return read_io_remfile(s3_path), "remfile" def read_nwbfile(io: NWBHDF5IO) -> NWBFile: @@ -118,6 +121,33 @@ def read_slice(nwb_object: Any, start: List[int], count: List[int]) -> Any: return dataset[tuple(slices)] + +def format_benchmark_result( + requested_driver: str, + actual_driver: str, + timings_seconds: Dict[str, float], + data_size_elements: int, +) -> Dict[str, Any]: + return { + "implementation": "python", + "requested_driver": requested_driver, + "actual_driver": actual_driver, + "timings_seconds": timings_seconds, + "data_size_elements": data_size_elements, + } + + +def print_text_result(result: Dict[str, Any]) -> None: + print("Benchmarking remote read process (Python using PyNWB)...") + print(f"Requested driver: {result['requested_driver']}") + print(f"Actual driver: {result['actual_driver']}") + print(f"read_io took: {result['timings_seconds']['read_io']:.6f} s") + print(f"read_nwbfile took: {result['timings_seconds']['read_nwbfile']:.6f} s") + print(f"find_object took: {result['timings_seconds']['find_object']:.6f} s") + print(f"read_slice took: {result['timings_seconds']['read_slice']:.6f} s") + print(f"Total time taken: {result['timings_seconds']['total']:.6f} s") + print(f"Data read size: {result['data_size_elements']} elements") + def main() -> None: parser = argparse.ArgumentParser(description="ROS3 read process benchmark (Python version)") parser.add_argument("s3_path", help="S3 URL of the NWB file") @@ -125,14 +155,36 @@ def main() -> None: parser.add_argument("object_name", help="Name of the object to find") parser.add_argument("start_indices", help="Comma-separated start indices (e.g., '0,0')") parser.add_argument("count_indices", help="Comma-separated count indices (e.g., '10,1')") + parser.add_argument( + "--driver", + choices=("ros3", "remfile"), + default="ros3", + help="Requested driver to benchmark. Defaults to ros3.", + ) parser.add_argument( "--force-remfile", action="store_true", help="Force the use of remfile instead of the ROS3 driver, even if ROS3 is available.", ) + parser.add_argument( + "--strict-driver", + action="store_true", + help="Fail instead of falling back to remfile when the requested driver is unavailable.", + ) + parser.add_argument( + "--output-format", + choices=("text", "json"), + default="text", + help="Choose human-readable text or machine-readable JSON output.", + ) args = parser.parse_args() + if args.force_remfile: + requested_driver = "remfile" + else: + requested_driver = args.driver + try: start = [int(x) for x in args.start_indices.split(',')] count = [int(x) for x in args.count_indices.split(',')] @@ -141,40 +193,54 @@ def main() -> None: exit(1) try: - print("Benchmarking ROS3 read process (Python using PyNWB)...") - total_start = time.perf_counter() - + # 1. read_io io_start = time.perf_counter() - io = read_io(args.s3_path, args.aws_region, force_remfile=args.force_remfile) + io, actual_driver = read_io( + args.s3_path, + args.aws_region, + driver=requested_driver, + strict_driver=args.strict_driver, + ) io_end = time.perf_counter() - print(f"read_io took: {io_end - io_start:.6f} s") - + # 2. read_nwbfile nwb_start = time.perf_counter() nwb = read_nwbfile(io) nwb_end = time.perf_counter() - print(f"read_nwbfile took: {nwb_end - nwb_start:.6f} s") - + # 3. find_object find_start = time.perf_counter() nwb_object = get_object_by_name(nwb, args.object_name) find_end = time.perf_counter() - print(f"find_object took: {find_end - find_start:.6f} s") - + # 4. read_slice slice_start = time.perf_counter() data = read_slice(nwb_object, start, count) slice_end = time.perf_counter() - print(f"read_slice took: {slice_end - slice_start:.6f} s") - + total_end = time.perf_counter() - print(f"Total time taken: {total_end - total_start:.6f} s") - print(f"Data read size: {data.size} elements") - + result = format_benchmark_result( + requested_driver=requested_driver, + actual_driver=actual_driver, + timings_seconds={ + "read_io": io_end - io_start, + "read_nwbfile": nwb_end - nwb_start, + "find_object": find_end - find_start, + "read_slice": slice_end - slice_start, + "total": total_end - total_start, + }, + data_size_elements=int(data.size), + ) + + if args.output_format == "json": + print(json.dumps(result)) + else: + print_text_result(result) + io.close() - + except Exception as e: print(f"Error: {e}") import traceback diff --git a/demo/remote_read_benchmark/main.cpp b/demo/remote_read_benchmark/main.cpp index e3c12084d..b88eec0d2 100644 --- a/demo/remote_read_benchmark/main.cpp +++ b/demo/remote_read_benchmark/main.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,19 @@ using namespace AQNWB; using namespace AQNWB::IO; using namespace AQNWB::NWB; +struct BenchmarkResult +{ + std::string implementation; + std::string requestedDriver; + std::string actualDriver; + double readIoSeconds; + double readNwbFileSeconds; + double findObjectSeconds; + double readSliceSeconds; + double totalSeconds; + size_t dataSizeElements; +}; + /** * @brief Creates the HDF5IO object and opens the file with the * requested driver ("ros3" or "remfile"). @@ -145,7 +159,7 @@ void printUsage(const char* programName) { std::cout << "Usage: " << programName << " " - " [driver]" + " [driver] [--json]" << std::endl; std::cout << "Example: " << programName << " https://bucket.s3.amazonaws.com/file us-east-1 my_timeseries " @@ -189,6 +203,40 @@ SizeArray parseIndices(const std::string& s) return indices; } +void printBenchmarkResultText(const BenchmarkResult& result) +{ + std::cout << "Benchmarking " << result.actualDriver << " read process..." + << std::endl; + std::cout << "Requested driver: " << result.requestedDriver << std::endl; + std::cout << "Actual driver: " << result.actualDriver << std::endl; + std::cout << "read_io took: " << result.readIoSeconds << " s" << std::endl; + std::cout << "read_nwbfile took: " << result.readNwbFileSeconds << " s" + << std::endl; + std::cout << "find_object took: " << result.findObjectSeconds << " s" + << std::endl; + std::cout << "read_slice took: " << result.readSliceSeconds << " s" + << std::endl; + std::cout << "Total time taken: " << result.totalSeconds << " s" << std::endl; + std::cout << "Data read size: " << result.dataSizeElements << " elements" + << std::endl; +} + +void printBenchmarkResultJson(const BenchmarkResult& result) +{ + std::cout << std::fixed << std::setprecision(6) << "{" + << "\"implementation\":\"" << result.implementation << "\"," + << "\"requested_driver\":\"" << result.requestedDriver << "\"," + << "\"actual_driver\":\"" << result.actualDriver << "\"," + << "\"timings_seconds\":{" + << "\"read_io\":" << result.readIoSeconds << "," + << "\"read_nwbfile\":" << result.readNwbFileSeconds << "," + << "\"find_object\":" << result.findObjectSeconds << "," + << "\"read_slice\":" << result.readSliceSeconds << "," + << "\"total\":" << result.totalSeconds << "}," + << "\"data_size_elements\":" << result.dataSizeElements << "}" + << std::endl; +} + int main(int argc, char* argv[]) { if (argc < 6) { @@ -201,50 +249,75 @@ int main(int argc, char* argv[]) std::string objectName = argv[3]; SizeArray start = parseIndices(argv[4]); SizeArray count = parseIndices(argv[5]); - std::string driver = (argc > 6) ? argv[6] : "ros3"; + std::string driver = "ros3"; + bool jsonOutput = false; - try { - std::cout << "Benchmarking " << driver << " read process..." << std::endl; + for (int i = 6; i < argc; ++i) { + std::string arg = argv[i]; + if (arg == "ros3" || arg == "remfile") { + driver = arg; + } else if (arg == "--json") { + jsonOutput = true; + } else { + std::cerr << "Unknown argument: " << arg << std::endl; + printUsage(argv[0]); + return 1; + } + } + try { auto start_total = std::chrono::high_resolution_clock::now(); // 1. read_io auto start_io = std::chrono::high_resolution_clock::now(); auto readio = read_io(s3Path, awsRegion, driver); auto end_io = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed_io = end_io - start_io; - std::cout << "read_io took: " << elapsed_io.count() << " s" << std::endl; + const double elapsedIoSeconds = + std::chrono::duration(end_io - start_io).count(); // 2. read_nwbfile auto start_nwb = std::chrono::high_resolution_clock::now(); auto nwbFile = read_nwbfile(readio); auto end_nwb = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed_nwb = end_nwb - start_nwb; - std::cout << "read_nwbfile took: " << elapsed_nwb.count() << " s" - << std::endl; + const double elapsedNwbSeconds = + std::chrono::duration(end_nwb - start_nwb).count(); // 3. find_object auto start_find = std::chrono::high_resolution_clock::now(); auto object = find_object(nwbFile, objectName); auto end_find = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed_find = end_find - start_find; - std::cout << "find_object took: " << elapsed_find.count() << " s" - << std::endl; + const double elapsedFindSeconds = + std::chrono::duration(end_find - start_find).count(); // 4. read_slice (Assuming int16_t for benchmark, can be adjusted or made an // argument) auto start_slice = std::chrono::high_resolution_clock::now(); auto data = read_slice(object, start, count); auto end_slice = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed_slice = end_slice - start_slice; - std::cout << "read_slice took: " << elapsed_slice.count() << " s" - << std::endl; + const double elapsedSliceSeconds = + std::chrono::duration(end_slice - start_slice).count(); auto end_total = std::chrono::high_resolution_clock::now(); - std::chrono::duration elapsed_total = end_total - start_total; - std::cout << "Total time taken: " << elapsed_total.count() << " s" - << std::endl; - std::cout << "Data read size: " << data.size() << " elements" << std::endl; + const double elapsedTotalSeconds = + std::chrono::duration(end_total - start_total).count(); + + BenchmarkResult result { + "cpp", + driver, + driver, + elapsedIoSeconds, + elapsedNwbSeconds, + elapsedFindSeconds, + elapsedSliceSeconds, + elapsedTotalSeconds, + data.size(), + }; + + if (jsonOutput) { + printBenchmarkResultJson(result); + } else { + printBenchmarkResultText(result); + } readio->close(); } catch (const std::exception& e) { diff --git a/demo/remote_read_benchmark/run_benchmark_matrix.py b/demo/remote_read_benchmark/run_benchmark_matrix.py new file mode 100644 index 000000000..0ef5c3d12 --- /dev/null +++ b/demo/remote_read_benchmark/run_benchmark_matrix.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +""" +Run the remote read benchmark repeatedly for all C++/Python and ROS3/remfile cases +within a single HDF5 environment and write the raw results to JSON. +""" + +import argparse +import json +import subprocess +from pathlib import Path +from typing import Any, Dict, List + + +def build_case_commands(args: argparse.Namespace) -> List[Dict[str, Any]]: + return [ + { + "implementation": "cpp", + "driver": "ros3", + "command": [ + str(args.cpp_binary), + args.s3_path, + args.aws_region, + args.object_name, + args.start_indices, + args.count_indices, + "ros3", + "--json", + ], + }, + { + "implementation": "cpp", + "driver": "remfile", + "command": [ + str(args.cpp_binary), + args.s3_path, + args.aws_region, + args.object_name, + args.start_indices, + args.count_indices, + "remfile", + "--json", + ], + }, + { + "implementation": "python", + "driver": "ros3", + "command": [ + args.python_executable, + str(args.python_script), + args.s3_path, + args.aws_region, + args.object_name, + args.start_indices, + args.count_indices, + "--driver", + "ros3", + "--strict-driver", + "--output-format", + "json", + ], + }, + { + "implementation": "python", + "driver": "remfile", + "command": [ + args.python_executable, + str(args.python_script), + args.s3_path, + args.aws_region, + args.object_name, + args.start_indices, + args.count_indices, + "--driver", + "remfile", + "--strict-driver", + "--output-format", + "json", + ], + }, + ] + + +def run_case(case: Dict[str, Any], iteration: int, hdf5_version: str) -> Dict[str, Any]: + completed = subprocess.run( + case["command"], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"Benchmark failed for {case['implementation']}/{case['driver']} run {iteration} " + f"(exit code {completed.returncode}).\nSTDOUT:\n{completed.stdout}\nSTDERR:\n{completed.stderr}" + ) + + stdout = completed.stdout.strip() + try: + payload = json.loads(stdout) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"Benchmark output was not valid JSON for {case['implementation']}/{case['driver']} " + f"run {iteration}.\nSTDOUT:\n{stdout}\nSTDERR:\n{completed.stderr}" + ) from exc + + if payload["implementation"] != case["implementation"]: + raise RuntimeError( + f"Expected implementation {case['implementation']}, got {payload['implementation']} " + f"for run {iteration}." + ) + if payload["requested_driver"] != case["driver"] or payload["actual_driver"] != case["driver"]: + raise RuntimeError( + f"Driver mismatch for {case['implementation']}/{case['driver']} run {iteration}: " + f"requested={payload['requested_driver']} actual={payload['actual_driver']}." + ) + + payload["hdf5_version"] = hdf5_version + payload["iteration"] = iteration + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run all remote read benchmark variants repeatedly.") + parser.add_argument("--cpp-binary", type=Path, required=True, help="Path to the C++ benchmark executable.") + parser.add_argument("--python-script", type=Path, required=True, help="Path to benchmark.py.") + parser.add_argument("--python-executable", default="python", help="Python executable used to run benchmark.py.") + parser.add_argument("--hdf5-version", required=True, help="Label for the HDF5 version under test.") + parser.add_argument("--repetitions", type=int, default=10, help="Number of repetitions per case.") + parser.add_argument("--output-json", type=Path, required=True, help="Path to the output JSON file.") + parser.add_argument("s3_path", help="Remote NWB file URL.") + parser.add_argument("aws_region", help="AWS region for ROS3.") + parser.add_argument("object_name", help="Name of the NWB object to read.") + parser.add_argument("start_indices", help="Comma-separated slice start indices.") + parser.add_argument("count_indices", help="Comma-separated slice counts.") + args = parser.parse_args() + + if args.repetitions < 1: + raise ValueError("--repetitions must be at least 1") + + cases = build_case_commands(args) + results = [] + for case in cases: + for iteration in range(1, args.repetitions + 1): + print( + f"Running HDF5 {args.hdf5_version}: {case['implementation']}/{case['driver']} " + f"iteration {iteration}/{args.repetitions}", + flush=True, + ) + results.append(run_case(case, iteration, args.hdf5_version)) + + output_payload = { + "benchmark_suite": "remote_read_benchmark", + "hdf5_version": args.hdf5_version, + "repetitions": args.repetitions, + "target": { + "s3_path": args.s3_path, + "aws_region": args.aws_region, + "object_name": args.object_name, + "start_indices": args.start_indices, + "count_indices": args.count_indices, + }, + "results": results, + } + + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps(output_payload, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/demo/remote_read_benchmark/summarize_benchmark_results.py b/demo/remote_read_benchmark/summarize_benchmark_results.py new file mode 100644 index 000000000..d775855bc --- /dev/null +++ b/demo/remote_read_benchmark/summarize_benchmark_results.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 + +""" +Combine benchmark JSON files into markdown summary tables for GitHub Actions. +""" + +import argparse +import csv +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List, Tuple + + +def collect_results(json_files: Iterable[Path]) -> List[Dict[str, Any]]: + results: List[Dict[str, Any]] = [] + for json_file in sorted(json_files): + payload = json.loads(json_file.read_text(encoding="utf-8")) + results.extend(payload["results"]) + return results + + +def render_table(headers: List[str], rows: List[List[str]]) -> str: + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join(["---"] * len(headers)) + " |", + ] + lines.extend("| " + " | ".join(row) + " |" for row in rows) + return "\n".join(lines) + + +def format_seconds(value: float) -> str: + return f"{value:.6f}" + + +def get_fastest_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + fastest: Dict[Tuple[str, str, str], Dict[str, Any]] = {} + for result in results: + key = (result["hdf5_version"], result["implementation"], result["actual_driver"]) + current = fastest.get(key) + if current is None or result["timings_seconds"]["total"] < current["timings_seconds"]["total"]: + fastest[key] = result + return [ + fastest[key] + for key in sorted(fastest, key=lambda item: (item[0], item[1], item[2])) + ] + + +def build_rows(results: List[Dict[str, Any]]) -> List[List[str]]: + return [ + [ + result["hdf5_version"], + result["implementation"], + result["actual_driver"], + str(result["iteration"]), + format_seconds(result["timings_seconds"]["read_io"]), + format_seconds(result["timings_seconds"]["read_nwbfile"]), + format_seconds(result["timings_seconds"]["find_object"]), + format_seconds(result["timings_seconds"]["read_slice"]), + format_seconds(result["timings_seconds"]["total"]), + str(result["data_size_elements"]), + ] + for result in results + ] + + +def write_csv(output_path: Path, headers: List[str], rows: List[List[str]]) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8", newline="") as file: + writer = csv.writer(file) + writer.writerow(headers) + writer.writerows(rows) + + +def build_markdown(results: List[Dict[str, Any]]) -> str: + sorted_results = sorted( + results, + key=lambda result: ( + result["hdf5_version"], + result["implementation"], + result["actual_driver"], + result["iteration"], + ), + ) + all_rows = build_rows(sorted_results) + fastest_rows = build_rows(get_fastest_results(sorted_results)) + + headers = [ + "HDF5", + "Implementation", + "Driver", + "Run", + "read_io (s)", + "read_nwbfile (s)", + "find_object (s)", + "read_slice (s)", + "total (s)", + "data size", + ] + + sections = [ + "# Remote Read Benchmark Summary", + "", + "## All runs", + "", + render_table(headers, all_rows), + "", + "## Fastest run per case", + "", + render_table(headers, fastest_rows), + "", + ] + return "\n".join(sections) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Summarize remote read benchmark JSON results.") + parser.add_argument( + "--results-dir", + type=Path, + required=True, + help="Directory containing benchmark JSON files.", + ) + parser.add_argument( + "--output-markdown", + type=Path, + required=True, + help="Path to the generated markdown summary.", + ) + parser.add_argument( + "--output-json", + type=Path, + help="Optional path to the merged raw JSON results.", + ) + parser.add_argument( + "--output-all-runs-csv", + type=Path, + help="Optional CSV path for the full results table.", + ) + parser.add_argument( + "--output-fastest-runs-csv", + type=Path, + help="Optional CSV path for the fastest-runs table.", + ) + args = parser.parse_args() + + json_files = sorted(args.results_dir.glob("benchmark-results-*.json")) + if not json_files: + raise FileNotFoundError(f"No benchmark result files found in {args.results_dir}") + + results = collect_results(json_files) + sorted_results = sorted( + results, + key=lambda result: ( + result["hdf5_version"], + result["implementation"], + result["actual_driver"], + result["iteration"], + ), + ) + headers = [ + "HDF5", + "Implementation", + "Driver", + "Run", + "read_io (s)", + "read_nwbfile (s)", + "find_object (s)", + "read_slice (s)", + "total (s)", + "data size", + ] + markdown = build_markdown(results) + + args.output_markdown.parent.mkdir(parents=True, exist_ok=True) + args.output_markdown.write_text(markdown, encoding="utf-8") + + if args.output_json is not None: + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps({"results": results}, indent=2) + "\n", encoding="utf-8") + + if args.output_all_runs_csv is not None: + write_csv(args.output_all_runs_csv, headers, build_rows(sorted_results)) + + if args.output_fastest_runs_csv is not None: + write_csv(args.output_fastest_runs_csv, headers, build_rows(get_fastest_results(sorted_results))) + + +if __name__ == "__main__": + main()