Benchmark for remote read

.github/workflows/performance-analysis.yml

@@ -0,0 +1,170 @@
+name: Remote Read Performance Analysis
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+
+env:
+  BENCHMARK_S3_PATH: "https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a"
+  BENCHMARK_AWS_REGION: "us-east-2"
+  BENCHMARK_OBJECT_NAME: "ElectricalSeriesAp"
+  BENCHMARK_START_INDICES: "0,0"
+  BENCHMARK_COUNT_INDICES: "10,1"
+  BENCHMARK_REPETITIONS: "10"
+
+jobs:
+  benchmark:
+    name: HDF5 ${{ matrix.hdf5 }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - hdf5: "1.14"
+            conda_hdf5_spec: "hdf5=1.14"
+            hdf5_root: "$CONDA_PREFIX"
+          - hdf5: "2.2"
+            conda_hdf5_spec: ""
+            hdf5_root: "$RUNNER_TEMP/hdf5-install"
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Conda
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        activate-environment: aqnwb-performance
+        channels: conda-forge
+        conda-remove-defaults: true
+
+    - name: Install benchmark dependencies
+      run: |
+        conda install -y \
+          python=3.12 \
+          cmake \
+          ninja \
+          numpy \
+          libcurl \
+          pip
+        if [ -n "${{ matrix.conda_hdf5_spec }}" ]; then
+          conda install -y "${{ matrix.conda_hdf5_spec }}"
+        fi
+        if [ "${{ matrix.hdf5 }}" = "2.2" ]; then
+          conda install -y aws-c-s3
+        fi
+
+    - name: Build HDF5 2.2 from source
+      if: matrix.hdf5 == '2.2'
+      run: |
+        git clone --depth 1 --branch hdf5_2_2_0 https://github.com/HDFGroup/hdf5.git hdf5-src
+        cmake -S hdf5-src -B hdf5-build \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX="$RUNNER_TEMP/hdf5-install" \
+          -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" \
+          -DHDF5_ENABLE_ROS3_VFD=ON \
+          -DHDF5_BUILD_CPP_LIB=ON \
+          -DHDF5_BUILD_TOOLS=OFF \
+          -DHDF5_BUILD_EXAMPLES=OFF \
+          -DBUILD_TESTING=OFF \
+          -DBUILD_SHARED_LIBS=ON
+        cmake --build hdf5-build --config Release -j 2
+        cmake --install hdf5-build
+
+    - name: Install Python benchmark stack against selected HDF5
+      run: |
+        export HDF5_DIR="${{ matrix.hdf5_root }}"
+        export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}"
+        python -m pip install --no-binary=h5py h5py
+        python -m pip install pynwb remfile
+
+    - name: Configure and build aqnwb
+      run: |
+        export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}"
+        cmake -S . -B build \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DAQNWB_USE_REMFILE=ON \
+          -DCMAKE_PREFIX_PATH="${{ matrix.hdf5_root }};$CONDA_PREFIX" \
+          -DHDF5_ROOT="${{ matrix.hdf5_root }}" \
+          -DHDF5_USE_STATIC_LIBRARIES=OFF
+        cmake --build build --config Release -j 2
+        cmake --install build --prefix "$RUNNER_TEMP/aqnwb-install"
+
+    - name: Build remote_read_benchmark demo
+      run: |
+        export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}"
+        cmake -S demo/remote_read_benchmark -B demo/remote_read_benchmark/build \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_PREFIX_PATH="$RUNNER_TEMP/aqnwb-install;${{ matrix.hdf5_root }};$CONDA_PREFIX" \
+          -DHDF5_ROOT="${{ matrix.hdf5_root }}" \
+          -DHDF5_USE_STATIC_LIBRARIES=OFF
+        cmake --build demo/remote_read_benchmark/build --config Release -j 2
+
+    - name: Run benchmark suite
+      run: |
+        export LD_LIBRARY_PATH="${{ matrix.hdf5_root }}/lib:$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}"
+        python demo/remote_read_benchmark/run_benchmark_matrix.py \
+          --cpp-binary demo/remote_read_benchmark/build/bin/remote_read_benchmark \
+          --python-script demo/remote_read_benchmark/benchmark.py \
+          --python-executable python \
+          --hdf5-version "${{ matrix.hdf5 }}" \
+          --repetitions "$BENCHMARK_REPETITIONS" \
+          --output-json "artifacts/benchmark-results-${{ matrix.hdf5 }}.json" \
+          "$BENCHMARK_S3_PATH" \
+          "$BENCHMARK_AWS_REGION" \
+          "$BENCHMARK_OBJECT_NAME" \
+          "$BENCHMARK_START_INDICES" \
+          "$BENCHMARK_COUNT_INDICES"
+
+    - name: Upload benchmark results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-results-hdf5-${{ matrix.hdf5 }}
+        path: artifacts/benchmark-results-${{ matrix.hdf5 }}.json
+
+  summarize:
+    name: Summarize benchmark results
+    needs: benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Download benchmark artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: benchmark-artifacts
+        pattern: benchmark-results-hdf5-*
+        merge-multiple: true
+
+    - name: Generate markdown summary
+      run: |
+        python demo/remote_read_benchmark/summarize_benchmark_results.py \
+          --results-dir benchmark-artifacts \
+          --output-markdown benchmark-artifacts/benchmark-summary.md \
+          --output-json benchmark-artifacts/benchmark-summary.json \
+          --output-all-runs-csv benchmark-artifacts/benchmark-all-runs.csv \
+          --output-fastest-runs-csv benchmark-artifacts/benchmark-fastest-runs.csv
+        cat benchmark-artifacts/benchmark-summary.md >> "$GITHUB_STEP_SUMMARY"
+        echo "===== Remote Read Benchmark Summary (Markdown) ====="
+        cat benchmark-artifacts/benchmark-summary.md
+        echo "===== All Runs CSV ====="
+        cat benchmark-artifacts/benchmark-all-runs.csv
+        echo "===== Fastest Runs CSV ====="
+        cat benchmark-artifacts/benchmark-fastest-runs.csv
+
+    - name: Upload benchmark summary
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-summary
+        path: |
+          benchmark-artifacts/benchmark-summary.md
+          benchmark-artifacts/benchmark-summary.json
+          benchmark-artifacts/benchmark-all-runs.csv
+          benchmark-artifacts/benchmark-fastest-runs.csv

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     * Added `HDF5IO::openRemote()` method to read remote NWB files over HTTP(S) using the [remfile-cpp](https://github.com/catalystneuro/remfile-cpp) virtual file driver (a C++ port of the Python [remfile](https://github.com/magland/remfile) package), imported as an optional CMake dependency (`AQNWB_USE_REMFILE`, requires `libcurl`). Unlike ROS3, remfile does not require HDF5 to be built with ROS3 support and works with any HTTP(S) server that supports byte-range requests. (@bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309))
     * Added demo for benchmarking ROS3 and remfile performance and comparing with PyNWB S3 reads (`demo/remote_read_benchmark`). (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309))
     * Added tutorial on using the ROS3 and remfile drivers to read NWB files in S3 (`docs/pages/userdocs/reads3.dox`) (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308); @bendichter, [#309](https://github.com/NeurodataWithoutBorders/aqnwb/pull/309))
+    * Added a dedicated `performance-analysis.yml` workflow plus benchmark helper scripts to run the remote-read benchmarks repeatedly across HDF5 1.14 and 2.2 and publish markdown summary tables. (@copilot)
 * Added `ElectricalSeries::writeAllChannels` method and `IO::writeElectricalSeriesData` overload to simplify zero-copy interleaved multichannel writes. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293))
 * Added `ElectricalSeries::channelsAtSameSampleOffset` method to check if all channels are at the same sample offset, which is a requirement for using `writeAllChannels`. (@copilot, @oruebel, [#293](https://github.com/NeurodataWithoutBorders/aqnwb/pull/293))
 * Added new `BaseIO::findObject` and `RegisteredType::findOwnedObject` methods to simplify searching for objects by name. Added `HDF5IO::findObject` override method to optimize the search for HDF5 objects. (@oruebel, [#308](https://github.com/NeurodataWithoutBorders/aqnwb/pull/308))   

demo/remote_read_benchmark/README.md

@@ -115,8 +115,37 @@ By default, the script will attempt to use the ROS3 driver (falling back to
 `remfile` instead of the ROS3 driver (e.g. to compare the performance of
 the two read strategies), pass the `--force-remfile` flag.
 
+For automation, both the C++ and Python benchmarks also support machine-readable
+JSON output:
+
+```bash
+./remote_read_benchmark \
+    "https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a" \
+    "us-east-2" \
+    "ElectricalSeriesAp" \
+    "0,0" \
+    "10,1" \
+    --json
+
+python demo/remote_read_benchmark/benchmark.py \
+    "https://dandiarchive.s3.amazonaws.com/blobs/fec/8a6/fec8a690-2ece-4437-8877-8a002ff8bd8a" \
+    "us-east-2" \
+    "ElectricalSeriesAp" \
+    "0,0" \
+    "10,1" \
+    --driver ros3 \
+    --strict-driver \
+    --output-format json
+```
+
+The helper scripts `run_benchmark_matrix.py` and `summarize_benchmark_results.py`
+are used by the dedicated GitHub Actions performance workflow to run all
+benchmark variants repeatedly and render markdown summary tables.
+
 ## Code Structure
 
 - `main.cpp`: Contains the C++ benchmarking logic and timing measurements.
 - `CMakeLists.txt`: CMake configuration file for building the C++ project.
 - `benchmark.py`: Contains the Python benchmarking logic using PyNWB.
+- `run_benchmark_matrix.py`: Repeats all benchmark variants for one HDF5 environment and writes raw JSON results.
+- `summarize_benchmark_results.py`: Combines JSON artifacts into markdown summary tables.