diff --git a/docs/gallery/advanced_io/streaming.py b/docs/gallery/advanced_io/streaming.py index 47d1dc940..8d2f1586a 100644 --- a/docs/gallery/advanced_io/streaming.py +++ b/docs/gallery/advanced_io/streaming.py @@ -169,6 +169,48 @@ # One downside of the fsspec method is that fsspec is not optimized for reading HDF5 files, and so streaming data # using this method can be slow. ``remfile`` may be a faster alternative. # +# Streaming data with LINDI +# ------------------------- +# The Linked Data Interface (LINDI) provides a cloud-friendly representation of NWB data. It stores the +# metadata and references to data chunks separately, so a remote NWB file can be opened without first +# downloading the complete HDF5 file. LINDI exposes an h5py-compatible file object, which means that it +# can be passed directly to :py:class:`~pynwb.NWBHDF5IO`. +# +# Install LINDI with pip: +# +# .. code-block:: bash +# +# pip install lindi +# +# A local cache is optional, but is useful when exploring the same remote file repeatedly. The cache stores +# chunks after their first download and can be reused across Python sessions. + +import lindi + +local_cache = lindi.LocalCache() + +with lindi.LindiH5pyFile.from_hdf5_file(s3_url, local_cache=local_cache) as lindi_file: + with NWBHDF5IO(file=lindi_file, mode="r") as io: + nwbfile = io.read() + streamed_data = nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:] + +################################## +# LINDI reference files can also be opened directly from a local path or a URL. A reference file stores the +# remote data locations in a compact ``.lindi.json`` document, so it can be generated once and reused later: +# +# .. code-block:: python +# +# with lindi.LindiH5pyFile.from_lindi_file( +# "https://example.org/recording.nwb.lindi.json", +# local_cache=local_cache, +# ) as lindi_file: +# with NWBHDF5IO(file=lindi_file, mode="r") as io: +# nwbfile = io.read() +# streamed_data = nwbfile.acquisition["lick_times"].time_series["lick_left_times"].data[:] +# +# As with the other methods in this tutorial, data are loaded lazily. Slicing a dataset downloads only the +# requested portion, while the metadata is kept in memory. + # Streaming data with ROS3 # ------------------------ # ROS3 stands for "read only S3" and is a driver created by the HDF5 Group that allows HDF5 to read HDF5 files stored diff --git a/pyproject.toml b/pyproject.toml index 886e18e64..93f7f6be9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ stream = [ "fsspec>=2023.1.0", "requests>=2.28.0", "aiohttp>=3.8.3", + "lindi>=0.4.6", ] # documentation dependencies. the gallery also needs the zarr and termset feature extras, which are @@ -90,6 +91,7 @@ docs = [ "lxml>=4.9.3", # used by dataframe_image when using the matplotlib backend "hdf5plugin>=4.0.0", "dandi>=0.76.5", # 0.76.5 lifts the click<8.2 bound, allowing linkml/linkml-runtime>=1.11 + "lindi>=0.4.6", # used by the streaming gallery example "pooch>=1.6.0", ]