From bd659c53645cedd1b298e50c9043324d6740c8ec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:04:22 +0000 Subject: [PATCH 1/2] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.5 → v0.16.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.5...v0.16.4) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c860b40..ef1ae06 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.14.5 + rev: v0.16.4 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 91c56f60ee4c7e9eb3203bace4946bfd927d92f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:04:46 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 2 +- src/cellarr_array/core/base.py | 41 +++++++++---------- src/cellarr_array/core/dense.py | 7 ++-- src/cellarr_array/core/helpers.py | 17 ++++---- src/cellarr_array/core/sparse.py | 26 ++++++------ src/cellarr_array/dataloaders/denseloader.py | 11 +++-- .../dataloaders/iterabledataloader.py | 12 +++--- src/cellarr_array/dataloaders/sparseloader.py | 11 +++-- src/cellarr_array/utils/__init__.py | 2 +- src/cellarr_array/utils/config.py | 12 +++--- src/cellarr_array/utils/mock.py | 5 +-- 11 files changed, 69 insertions(+), 77 deletions(-) diff --git a/setup.py b/setup.py index 51445f4..37a384d 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ if __name__ == "__main__": try: setup(use_scm_version={"version_scheme": "no-guess-dev"}) - except: # noqa + except: print( "\n\nAn error occurred while building the project, " "please ensure you have the most updated version of setuptools, " diff --git a/src/cellarr_array/core/base.py b/src/cellarr_array/core/base.py index 24dcd29..a45678d 100644 --- a/src/cellarr_array/core/base.py +++ b/src/cellarr_array/core/base.py @@ -6,7 +6,7 @@ except ImportError: # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025 EllipsisType = type(...) -from typing import Any, List, Literal, Optional, Tuple, Union +from typing import Any, Literal import numpy as np import tiledb @@ -25,11 +25,11 @@ class CellArray(ABC): def __init__( self, - uri: Optional[str] = None, - tiledb_array_obj: Optional[tiledb.Array] = None, + uri: str | None = None, + tiledb_array_obj: tiledb.Array | None = None, attr: str = "data", - mode: Optional[Literal["r", "w", "d", "m"]] = None, - config_or_context: Optional[Union[tiledb.Config, tiledb.Ctx]] = None, + mode: Literal["r", "w", "d", "m"] | None = None, + config_or_context: tiledb.Config | tiledb.Ctx | None = None, validate: bool = True, ): """Initialize the object. @@ -141,14 +141,14 @@ def _validate(self, attr): ) @property - def mode(self) -> Optional[str]: + def mode(self) -> str | None: """Get current array mode. If an external array is used, this is its open mode.""" if self._array_passed_in and self._opened_array_external is not None: return self._opened_array_external.mode return self._mode @mode.setter - def mode(self, value: Optional[str]): + def mode(self, value: str | None): """Set array mode for subsequent operations if not using an external array. This action does not affect an already passed-in external array's mode. @@ -167,7 +167,7 @@ def mode(self, value: Optional[str]): self._mode = value @property - def dim_names(self) -> List[str]: + def dim_names(self) -> list[str]: """Get dimension names of the array.""" if self._dim_names is None: with self.open_array(mode="r") as A: @@ -176,7 +176,7 @@ def dim_names(self) -> List[str]: return self._dim_names @property - def attr_names(self) -> List[str]: + def attr_names(self) -> list[str]: """Get attribute names of the array.""" if self._attr_names is None: with self.open_array(mode="r") as A: @@ -185,7 +185,7 @@ def attr_names(self) -> List[str]: return self._attr_names @property - def shape(self) -> Tuple[int, ...]: + def shape(self) -> tuple[int, ...]: if self._shape is None: with self.open_array(mode="r") as A: shape_list = [] @@ -202,7 +202,7 @@ def shape(self) -> Tuple[int, ...]: return self._shape @property - def nonempty_domain(self) -> Optional[Tuple[Any, ...]]: + def nonempty_domain(self) -> tuple[Any, ...] | None: if self._nonempty_domain is None: with self.open_array(mode="r") as A: # nonempty_domain() can return None if the array is empty. @@ -225,7 +225,7 @@ def ndim(self) -> int: return self._ndim @property - def dim_dtypes(self) -> List[np.dtype]: + def dim_dtypes(self) -> list[np.dtype]: """Get dimension dtypes of the array.""" if self._dim_dtypes is None: with self.open_array(mode="r") as A: @@ -234,7 +234,7 @@ def dim_dtypes(self) -> List[np.dtype]: return self._dim_dtypes @contextmanager - def open_array(self, mode: Optional[str] = None): + def open_array(self, mode: str | None = None): """Context manager for array operations. Uses the externally provided array if available, otherwise opens from URI. @@ -283,7 +283,7 @@ def open_array(self, mode: Optional[str] = None): finally: array.close() - def __getitem__(self, key: Union[slice, EllipsisType, Tuple[Union[slice, List[int]], ...], EllipsisType, str]): + def __getitem__(self, key: slice | EllipsisType | tuple[slice | list[int], ...] | str): """Get item implementation that routes to either direct slicing, multi_index, or query based on the type of indices provided. @@ -334,20 +334,18 @@ def __getitem__(self, key: Union[slice, EllipsisType, Tuple[Union[slice, List[in return self._multi_index(normalized_key) @abstractmethod - def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> np.ndarray: + def _direct_slice(self, key: tuple[slice | EllipsisType, ...]) -> np.ndarray: """Implementation for direct slicing.""" - pass @abstractmethod - def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> np.ndarray: + def _multi_index(self, key: tuple[slice | list[int], ...]) -> np.ndarray: """Implementation for multi-index access.""" - pass def vacuum(self) -> None: """Remove deleted fragments from the array.""" tiledb.vacuum(self.uri) - def consolidate(self, config: Optional[ConsolidationConfig] = None) -> None: + def consolidate(self, config: ConsolidationConfig | None = None) -> None: """Consolidate array fragments. Args: @@ -371,7 +369,7 @@ def consolidate(self, config: Optional[ConsolidationConfig] = None) -> None: self.vacuum() @abstractmethod - def write_batch(self, data: Union[np.ndarray, sparse.spmatrix], start_row: int, **kwargs) -> None: + def write_batch(self, data: np.ndarray | sparse.spmatrix, start_row: int, **kwargs) -> None: """Write a batch of data to the array starting at the specified row. Args: @@ -384,9 +382,8 @@ def write_batch(self, data: Union[np.ndarray, sparse.spmatrix], start_row: int, **kwargs: Additional arguments for write operation. """ - pass - def get_unique_dim_values(self, dim_name: Optional[str] = None) -> np.ndarray: + def get_unique_dim_values(self, dim_name: str | None = None) -> np.ndarray: """Get unique values for a dimension. Args: diff --git a/src/cellarr_array/core/dense.py b/src/cellarr_array/core/dense.py index a9c2f05..551b26e 100644 --- a/src/cellarr_array/core/dense.py +++ b/src/cellarr_array/core/dense.py @@ -3,7 +3,6 @@ except ImportError: # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025 EllipsisType = type(...) -from typing import List, Tuple, Union import numpy as np from scipy import sparse as sp @@ -19,7 +18,7 @@ class DenseCellArray(CellArray): """Implementation for dense TileDB arrays.""" - def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> np.ndarray: + def _direct_slice(self, key: tuple[slice | EllipsisType, ...]) -> np.ndarray: """Implementation for direct slicing of dense arrays. Args: @@ -33,7 +32,7 @@ def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> np.ndarr res = array[key] return res[self._attr] if self._attr is not None else res - def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> np.ndarray: + def _multi_index(self, key: tuple[slice | list[int], ...]) -> np.ndarray: """Implementation for multi-index access of dense arrays. Args: @@ -70,7 +69,7 @@ def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> np.ndarray: res = array.multi_index[tuple(tiledb_key)] return res[self._attr] if self._attr is not None else res - def write_batch(self, data: Union[np.ndarray, sp.spmatrix], start_row: int, **kwargs) -> None: + def write_batch(self, data: np.ndarray | sp.spmatrix, start_row: int, **kwargs) -> None: """Write a batch of data to the dense array. This method supports both dense (numpy.ndarray) and sparse diff --git a/src/cellarr_array/core/helpers.py b/src/cellarr_array/core/helpers.py index a378af4..b70e886 100644 --- a/src/cellarr_array/core/helpers.py +++ b/src/cellarr_array/core/helpers.py @@ -3,7 +3,6 @@ except ImportError: # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025 EllipsisType = type(...) -from typing import List, Optional, Tuple, Union import numpy as np import tiledb @@ -17,13 +16,13 @@ def create_cellarray( uri: str, - shape: Optional[Tuple[Optional[int], ...]] = None, - attr_dtype: Optional[Union[str, np.dtype]] = None, + shape: tuple[int | None, ...] | None = None, + attr_dtype: str | np.dtype | None = None, sparse: bool = False, mode: str = None, - config: Optional[CellArrConfig] = None, - dim_names: Optional[List[str]] = None, - dim_dtypes: Optional[List[Union[str, np.dtype]]] = None, + config: CellArrConfig | None = None, + dim_names: list[str] | None = None, + dim_dtypes: list[str | np.dtype] | None = None, attr_name: str = "data", **kwargs, ): @@ -165,7 +164,7 @@ class SliceHelper: """Helper class for handling array slicing operations.""" @staticmethod - def is_contiguous_indices(indices: List[int]) -> Optional[slice]: + def is_contiguous_indices(indices: list[int]) -> slice | None: """Checks if a list of indices is contiguous and can be converted to a slice. Returns None if the list is not contiguous or contains non-integers. @@ -191,10 +190,10 @@ def is_contiguous_indices(indices: List[int]) -> Optional[slice]: @staticmethod def normalize_index( - idx: Union[int, range, slice, List, str, EllipsisType], + idx: int | range | slice | list | str | EllipsisType, dim_size: int, dim_dtype: np.dtype, - ) -> Union[slice, List, EllipsisType]: + ) -> slice | list | EllipsisType: """Normalize index to handle negative indices and ensure consistency.""" is_string_dim = np.issubdtype(dim_dtype, np.str_) or np.issubdtype(dim_dtype, np.bytes_) diff --git a/src/cellarr_array/core/sparse.py b/src/cellarr_array/core/sparse.py index d23e69b..c7a2c35 100644 --- a/src/cellarr_array/core/sparse.py +++ b/src/cellarr_array/core/sparse.py @@ -3,7 +3,7 @@ except ImportError: # TODO: This is required for Python <3.10. Remove once Python 3.9 reaches EOL in October 2025 EllipsisType = type(...) -from typing import Dict, List, Literal, Optional, Tuple, Union +from typing import Literal import numpy as np import tiledb @@ -22,13 +22,13 @@ class SparseCellArray(CellArray): def __init__( self, - uri: Optional[str] = None, - tiledb_array_obj: Optional[tiledb.Array] = None, + uri: str | None = None, + tiledb_array_obj: tiledb.Array | None = None, attr: str = "data", - mode: Optional[Literal["r", "w", "d", "m"]] = None, - config_or_context: Optional[Union[tiledb.Config, tiledb.Ctx]] = None, + mode: Literal["r", "w", "d", "m"] | None = None, + config_or_context: tiledb.Config | tiledb.Ctx | None = None, return_sparse: bool = True, - sparse_format: Union[sparse.csr_matrix, sparse.csc_matrix] = sparse.csr_matrix, + sparse_format: sparse.csr_matrix | sparse.csc_matrix = sparse.csr_matrix, validate: bool = True, **kwargs, ): @@ -89,7 +89,7 @@ def __init__( self.sparse_format = sparse.csr_matrix if sparse_format is None else sparse_format self._list_remaps = {} - def _validate_matrix_dims(self, data: sparse.spmatrix) -> Tuple[sparse.coo_matrix, bool]: + def _validate_matrix_dims(self, data: sparse.spmatrix) -> tuple[sparse.coo_matrix, bool]: """Validate and adjust matrix dimensions if needed. Args: @@ -116,7 +116,7 @@ def _validate_matrix_dims(self, data: sparse.spmatrix) -> Tuple[sparse.coo_matri return coo_data, is_1d - def _get_slice_details(self, key: Tuple[Union[slice, List], ...]) -> ...: + def _get_slice_details(self, key: tuple[slice | list, ...]) -> ...: """Calculates the shape, remapping info, and if a remap is needed for a slice.""" shape = [] origins_or_maps = [] @@ -154,8 +154,8 @@ def _get_slice_details(self, key: Tuple[Union[slice, List], ...]) -> ...: return tuple(shape), origins_or_maps, is_list_remap def _to_sparse_format( - self, result: Dict[str, np.ndarray], key: Tuple[Union[slice, List[int]], ...] - ) -> Union[np.ndarray, sparse.spmatrix]: + self, result: dict[str, np.ndarray], key: tuple[slice | list[int], ...] + ) -> np.ndarray | sparse.spmatrix: """Convert TileDB result to CSR format or dense array.""" data = result[self._attr] @@ -197,7 +197,7 @@ def _to_sparse_format( return matrix - def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> Union[np.ndarray, sparse.coo_matrix]: + def _direct_slice(self, key: tuple[slice | EllipsisType, ...]) -> np.ndarray | sparse.coo_matrix: """Implementation for direct slicing of sparse arrays.""" self._list_remaps.clear() @@ -209,7 +209,7 @@ def _direct_slice(self, key: Tuple[Union[slice, EllipsisType], ...]) -> Union[np return self._to_sparse_format(result, key) - def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> Union[np.ndarray, sparse.coo_matrix]: + def _multi_index(self, key: tuple[slice | list[int], ...]) -> np.ndarray | sparse.coo_matrix: """Implementation for multi-index access of sparse arrays.""" self._list_remaps.clear() @@ -241,7 +241,7 @@ def _multi_index(self, key: Tuple[Union[slice, List[int]], ...]) -> Union[np.nda return self._to_sparse_format(result, key) def write_batch( - self, data: Union[sparse.spmatrix, sparse.csc_matrix, sparse.coo_matrix], start_row: int, **kwargs + self, data: sparse.spmatrix | sparse.csc_matrix | sparse.coo_matrix, start_row: int, **kwargs ) -> None: """Write a batch of sparse data to the array. diff --git a/src/cellarr_array/dataloaders/denseloader.py b/src/cellarr_array/dataloaders/denseloader.py index 6224936..239b3a7 100644 --- a/src/cellarr_array/dataloaders/denseloader.py +++ b/src/cellarr_array/dataloaders/denseloader.py @@ -1,4 +1,3 @@ -from typing import Optional from warnings import warn import numpy as np @@ -18,9 +17,9 @@ def __init__( self, array_uri: str, attribute_name: str = "data", - num_rows: Optional[int] = None, - num_columns: Optional[int] = None, - cellarr_ctx_config: Optional[dict] = None, + num_rows: int | None = None, + num_columns: int | None = None, + cellarr_ctx_config: dict | None = None, transform=None, ): """PyTorch Dataset for dense TileDB arrays accessed via DenseCellArray. @@ -142,8 +141,8 @@ def __getitem__(self, idx): def construct_dense_array_dataloader( array_uri: str, attribute_name: str = "data", - num_rows: Optional[int] = None, - num_columns: Optional[int] = None, + num_rows: int | None = None, + num_columns: int | None = None, batch_size: int = 1000, num_workers_dl: int = 2, ) -> DataLoader: diff --git a/src/cellarr_array/dataloaders/iterabledataloader.py b/src/cellarr_array/dataloaders/iterabledataloader.py index fa926a7..bd2ffae 100644 --- a/src/cellarr_array/dataloaders/iterabledataloader.py +++ b/src/cellarr_array/dataloaders/iterabledataloader.py @@ -1,4 +1,4 @@ -from typing import Callable, Dict, Iterator, Optional, Union +from collections.abc import Callable, Iterator import numpy as np import scipy.sparse as sp @@ -30,9 +30,9 @@ def __init__( num_columns: int, is_sparse: bool, batch_size: int = 1000, - num_yields_per_epoch_per_worker: Optional[int] = None, - cellarr_ctx_config: Optional[Dict] = None, - transform: Optional[Callable] = None, + num_yields_per_epoch_per_worker: int | None = None, + cellarr_ctx_config: dict | None = None, + transform: Callable | None = None, ): """Initializes the `CellArrayIterableDataset`. @@ -128,7 +128,7 @@ def _init_worker_state(self) -> None: uri=self.array_uri, attr=self.attribute_name, mode="r", config_or_context=ctx ) - def _fetch_one_random_batch(self) -> Union[np.ndarray, sp.spmatrix]: + def _fetch_one_random_batch(self) -> np.ndarray | sp.spmatrix: """Randomly selects `self.batch_size` row indices and fetches them from the TileDB array in a single multi-index read operation. @@ -163,7 +163,7 @@ def _fetch_one_random_batch(self) -> Union[np.ndarray, sp.spmatrix]: return data_chunk - def __iter__(self) -> Iterator[Union[np.ndarray, sp.spmatrix]]: + def __iter__(self) -> Iterator[np.ndarray | sp.spmatrix]: """Yields batches of randomly sampled data. This method is called by the DataLoader for each worker. diff --git a/src/cellarr_array/dataloaders/sparseloader.py b/src/cellarr_array/dataloaders/sparseloader.py index f03c6c9..06ea3bd 100644 --- a/src/cellarr_array/dataloaders/sparseloader.py +++ b/src/cellarr_array/dataloaders/sparseloader.py @@ -1,4 +1,3 @@ -from typing import Optional from warnings import warn import scipy.sparse as sp @@ -18,10 +17,10 @@ def __init__( self, array_uri: str, attribute_name: str = "data", - num_rows: Optional[int] = None, - num_columns: Optional[int] = None, + num_rows: int | None = None, + num_columns: int | None = None, sparse_format=sp.csr_matrix, - cellarr_ctx_config: Optional[dict] = None, + cellarr_ctx_config: dict | None = None, transform=None, ): """PyTorch Dataset for sparse TileDB arrays accessed via SparseCellArray. @@ -173,8 +172,8 @@ def sparse_coo_collate_fn(batch): def construct_sparse_array_dataloader( array_uri: str, attribute_name: str = "data", - num_rows: Optional[int] = None, - num_columns: Optional[int] = None, + num_rows: int | None = None, + num_columns: int | None = None, batch_size: int = 1000, num_workers_dl: int = 2, ) -> DataLoader: diff --git a/src/cellarr_array/utils/__init__.py b/src/cellarr_array/utils/__init__.py index d9d59f8..839716b 100644 --- a/src/cellarr_array/utils/__init__.py +++ b/src/cellarr_array/utils/__init__.py @@ -1,3 +1,3 @@ -from .config import CellArrConfig, ConsolidationConfig from ..core.helpers import create_cellarray +from .config import CellArrConfig, ConsolidationConfig # from .mock import generate_tiledb_dense_array, generate_tiledb_sparse_array diff --git a/src/cellarr_array/utils/config.py b/src/cellarr_array/utils/config.py index 740acf9..9a2f2c2 100644 --- a/src/cellarr_array/utils/config.py +++ b/src/cellarr_array/utils/config.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Any, Dict, List, Union +from typing import Any import tiledb @@ -15,13 +15,13 @@ class CellArrConfig: tile_capacity: int = 100000 cell_order: str = "row-major" tile_order: str = "row-major" - coords_filters: List[tiledb.Filter] = field(default_factory=lambda: [tiledb.LZ4Filter()]) - offsets_filters: List[tiledb.Filter] = field(default_factory=lambda: [tiledb.LZ4Filter()]) - attrs_filters: Dict[str, List[tiledb.Filter]] = field(default_factory=lambda: {"": [tiledb.LZ4Filter()]}) - ctx_config: Dict[str, Any] = field(default_factory=dict) + coords_filters: list[tiledb.Filter] = field(default_factory=lambda: [tiledb.LZ4Filter()]) + offsets_filters: list[tiledb.Filter] = field(default_factory=lambda: [tiledb.LZ4Filter()]) + attrs_filters: dict[str, list[tiledb.Filter]] = field(default_factory=lambda: {"": [tiledb.LZ4Filter()]}) + ctx_config: dict[str, Any] = field(default_factory=dict) @staticmethod - def create_filter(filter_config: Union[Dict[str, Any], tiledb.Filter]) -> tiledb.Filter: + def create_filter(filter_config: dict[str, Any] | tiledb.Filter) -> tiledb.Filter: """Create a TileDB Filter object from configuration.""" if isinstance(filter_config, tiledb.Filter): return filter_config diff --git a/src/cellarr_array/utils/mock.py b/src/cellarr_array/utils/mock.py index 8ae661a..26240f7 100644 --- a/src/cellarr_array/utils/mock.py +++ b/src/cellarr_array/utils/mock.py @@ -1,5 +1,4 @@ import shutil -from typing import Dict, Optional import numpy as np import scipy.sparse as sp @@ -20,7 +19,7 @@ def generate_tiledb_dense_array( attr_name: str = "data", attr_dtype: np.dtype = np.float32, chunk_size: int = 1000, - tiledb_config: Optional[Dict] = None, + tiledb_config: dict | None = None, ): """Generates a dense TileDB array and fills it with random float32 data. @@ -89,7 +88,7 @@ def generate_tiledb_sparse_array( attr_name: str = "data", attr_dtype: np.dtype = np.float32, chunk_size: int = 1000, - tiledb_config: Optional[Dict] = None, + tiledb_config: dict | None = None, sparse_format_to_write="coo", ): """Generates a sparse TileDB array and fills it with random float32 data.