diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py index d12085ec7fd..8ae41340ab7 100644 --- a/python/python/lance/dataset.py +++ b/python/python/lance/dataset.py @@ -3292,7 +3292,7 @@ def _prepare_scalar_index_request( if hasattr(field_type, "storage_type"): field_type = field_type.storage_type - if index_type in ["BTREE", "BITMAP", "ZONEMAP"]: + if index_type in ["BTREE", "BITMAP"]: if ( not pa.types.is_integer(field_type) and not pa.types.is_floating(field_type) @@ -3303,7 +3303,7 @@ def _prepare_scalar_index_request( and not pa.types.is_fixed_size_binary(field_type) ): raise TypeError( - f"BTREE/BITMAP/ZONEMAP index column {column} must be int", + f"BTREE/BITMAP index column {column} must be int", ", float, bool, str, large_str, fixed-size-binary, or temporal", ) elif index_type == "LABEL_LIST": diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py index 7c21d15fed0..50c153a56f7 100644 --- a/python/python/tests/test_scalar_index.py +++ b/python/python/tests/test_scalar_index.py @@ -2860,6 +2860,54 @@ def test_zonemap_index_remapping(tmp_path: Path): assert result.num_rows == 501 # 1000..1500 inclusive +def test_zonemap_fsl_column(tmp_path: Path): + """Zone map can be created on a FixedSizeList column and accelerates IS NULL.""" + dim = 8 + n = 1000 + rng = np.random.default_rng(42) + vectors = rng.standard_normal((n, dim)).astype(np.float32) + vec_type = pa.list_(pa.float32(), dim) + # Every 10th row is null + vec_list = [None if i % 10 == 0 else v.tolist() for i, v in enumerate(vectors)] + tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))}) + ds = lance.write_dataset(tbl, tmp_path) + ds.create_scalar_index("vec", index_type="ZONEMAP") + + scanner = ds.scanner(filter="vec IS NULL", prefilter=True) + plan = scanner.explain_plan() + assert "ScalarIndexQuery" in plan + result = scanner.to_table() + assert result.num_rows == 100 # every 10th row is null + + +def test_vector_and_zonemap_on_fsl_column(tmp_path: Path): + """Vector index and zone map can coexist on the same FSL column.""" + dim = 16 + n = 2000 + rng = np.random.default_rng(0) + vectors = rng.standard_normal((n, dim)).astype(np.float32) + vec_type = pa.list_(pa.float32(), dim) + # Every 20th row is null + vec_list = [None if i % 20 == 0 else v.tolist() for i, v in enumerate(vectors)] + tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))}) + ds = lance.write_dataset(tbl, tmp_path) + + ds.create_index("vec", index_type="IVF_PQ", num_partitions=4, num_sub_vectors=2) + ds.create_scalar_index("vec", index_type="ZONEMAP") + + # Vector search still works + query = vectors[5] + result = ds.scanner(nearest={"column": "vec", "q": query, "k": 10}).to_table() + assert result.num_rows == 10 + + # IS NULL is zone-map-accelerated + scanner = ds.scanner(filter="vec IS NULL", prefilter=True) + plan = scanner.explain_plan() + assert "ScalarIndexQuery" in plan + null_result = scanner.to_table() + assert null_result.num_rows == 100 # every 20th row is null + + def test_bloomfilter_index(tmp_path: Path): """Test create bloomfilter index""" tbl = pa.Table.from_arrays([pa.array([i for i in range(10000)])], names=["values"]) diff --git a/rust/lance-index-core/src/lib.rs b/rust/lance-index-core/src/lib.rs index 393064014e4..c949387b0b2 100644 --- a/rust/lance-index-core/src/lib.rs +++ b/rust/lance-index-core/src/lib.rs @@ -279,6 +279,31 @@ impl IndexType { .max() .unwrap_or(1) } + + pub fn matches_details(&self, details: &prost_types::Any) -> bool { + let url = &details.type_url; + match self { + Self::Scalar | Self::BTree => url.ends_with("BTreeIndexDetails"), + Self::Bitmap => url.ends_with("BitmapIndexDetails"), + Self::LabelList => url.ends_with("LabelListIndexDetails"), + Self::Inverted => url.ends_with("InvertedIndexDetails"), + Self::NGram => url.ends_with("NGramIndexDetails"), + Self::ZoneMap => url.ends_with("ZoneMapIndexDetails"), + Self::BloomFilter => url.ends_with("BloomFilterIndexDetails"), + Self::RTree => url.ends_with("RTreeIndexDetails"), + Self::Fm => url.ends_with("FMIndexDetails"), + Self::FragmentReuse => url.ends_with("FragmentReuseIndexDetails"), + Self::MemWal => url.ends_with("MemWalIndexDetails"), + Self::Vector + | Self::IvfFlat + | Self::IvfSq + | Self::IvfPq + | Self::IvfHnswSq + | Self::IvfHnswPq + | Self::IvfHnswFlat + | Self::IvfRq => url.ends_with("VectorIndexDetails"), + } + } } pub trait IndexParams: Send + Sync { diff --git a/rust/lance-index/src/registry.rs b/rust/lance-index/src/registry.rs index bd7448240fe..753f32afafd 100644 --- a/rust/lance-index/src/registry.rs +++ b/rust/lance-index/src/registry.rs @@ -16,6 +16,21 @@ use crate::{ }, }; +/// Derive the scalar index plugin name from a details type URL. +/// +/// Takes the last `.`-separated segment, lowercases it, and strips any trailing +/// `"indexdetails"` suffix so the result matches the plugin name used in +/// [`IndexPluginRegistry`]. For example, `/lance.index.pb.ZoneMapIndexDetails` +/// yields `"zonemap"`. +pub fn plugin_name_from_details_url(type_url: &str) -> String { + let segment = type_url.split('.').next_back().unwrap_or(type_url); + let lower = segment.to_lowercase(); + lower + .strip_suffix("indexdetails") + .map(|s| s.to_string()) + .unwrap_or(lower) +} + /// Derive a human-readable index type name from a details type URL. /// /// The display name is the final `.`-separated segment of the type URL with any @@ -42,12 +57,7 @@ impl IndexPluginRegistry { } fn get_plugin_name_from_details_name(&self, details_name: &str) -> String { - let details_name = Self::normalize_plugin_name(details_name); - if details_name.ends_with("indexdetails") { - details_name.replace("indexdetails", "") - } else { - details_name - } + plugin_name_from_details_url(details_name) } /// Adds a plugin to the registry, using the name of the details message to determine diff --git a/rust/lance-index/src/scalar/zonemap.rs b/rust/lance-index/src/scalar/zonemap.rs index b4a1d71261e..64ddf5618b4 100644 --- a/rust/lance-index/src/scalar/zonemap.rs +++ b/rust/lance-index/src/scalar/zonemap.rs @@ -183,19 +183,25 @@ impl ZoneMapIndex { ) -> Option<(ScalarValue, ScalarValue)> { let mut min: Option<&ScalarValue> = None; let mut max: Option<&ScalarValue> = None; - for zone in segments.into_iter().flat_map(|seg| seg.zones.iter()) { - if Self::scalar_is_nan(&zone.max) { + for seg in segments.into_iter() { + // Nested types have no meaningful ordering + if seg.data_type.is_nested() { return None; } - if Self::scalar_is_finite_bound(&zone.min) - && min.is_none_or(|cur| zone.min.partial_cmp(cur).is_some_and(|o| o.is_lt())) - { - min = Some(&zone.min); - } - if Self::scalar_is_finite_bound(&zone.max) - && max.is_none_or(|cur| zone.max.partial_cmp(cur).is_some_and(|o| o.is_gt())) - { - max = Some(&zone.max); + for zone in seg.zones.iter() { + if Self::scalar_is_nan(&zone.max) { + return None; + } + if Self::scalar_is_finite_bound(&zone.min) + && min.is_none_or(|cur| zone.min.partial_cmp(cur).is_some_and(|o| o.is_lt())) + { + min = Some(&zone.min); + } + if Self::scalar_is_finite_bound(&zone.max) + && max.is_none_or(|cur| zone.max.partial_cmp(cur).is_some_and(|o| o.is_gt())) + { + max = Some(&zone.max); + } } } Some((min?.clone(), max?.clone())) @@ -218,6 +224,29 @@ impl ZoneMapIndex { ) -> Result { use std::ops::Bound; + // For nested types we only track null_count; prune only when certain. + if self.data_type.is_nested() { + let all_null = zone.null_count as usize == zone.bound.length; + return match query { + SargableQuery::IsNull() => Ok(zone.null_count > 0), + SargableQuery::Equals(target) => { + if target.is_null() { + Ok(zone.null_count > 0) + } else { + Ok(!all_null) + } + } + SargableQuery::IsIn(values) => { + if values.iter().any(|v| !v.is_null()) { + Ok(!all_null) + } else { + Ok(zone.null_count > 0) + } + } + _ => Ok(!all_null), + }; + } + match query { SargableQuery::IsNull() => { // Zone contains matching values if it has any null values @@ -738,6 +767,10 @@ impl ScalarIndex for ZoneMapIndex { /// Single-segment `[min, max]` folded from this index's zones; see /// [`value_range_over`](Self::value_range_over) for the full contract. fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> { + // We don't record min/max for nested types + if self.data_type.is_nested() { + return None; + } Self::value_range_over([self]) } } @@ -1001,8 +1034,12 @@ impl ZoneMapIndexBuilder { } } -/// Index-specific processor that computes min/max statistics for each zone while the -/// trainer takes care of chunking and fragment boundaries. +/// Index-specific processor that computes zone statistics while the trainer +/// handles chunking and fragment boundaries. +/// +/// For non-nested types, tracks min, max, null_count, and nan_count. +/// For nested types (List, FixedSizeList, Struct, Map, etc.), tracks only +/// null_count; min and max are stored as typed null values. #[derive(Debug)] struct ZoneMapProcessor { data_type: DataType, @@ -1011,9 +1048,10 @@ struct ZoneMapProcessor { impl ZoneMapProcessor { fn new(data_type: DataType) -> Result { + let statistics = StatisticsAccumulator::new(&data_type); Ok(Self { - statistics: StatisticsAccumulator::new(&data_type), data_type, + statistics, }) } @@ -1072,6 +1110,19 @@ impl ZoneProcessor for ZoneMapProcessor { fn finish_zone(&mut self, bound: ZoneBound) -> Result { let statistics = self.statistics.statistics(); + let null_count = Self::stat_count_to_u32("null_count", statistics.null_count)?; + + // For nested types, only null_count is meaningful; store null min/max. + if self.data_type.is_nested() { + return Ok(ZoneMapStatistics { + min: ScalarValue::try_new_null(&self.data_type)?, + max: ScalarValue::try_new_null(&self.data_type)?, + null_count, + nan_count: 0, + bound, + }); + } + let nan_count = Self::stat_count_to_u32("nan_count", statistics.nan_count.unwrap_or(0))?; Ok(ZoneMapStatistics { min: Self::scalar_value_from_stat( @@ -1083,7 +1134,7 @@ impl ZoneProcessor for ZoneMapProcessor { &self.data_type, nan_count, )?, - null_count: Self::stat_count_to_u32("null_count", statistics.null_count)?, + null_count, nan_count, bound, }) @@ -1122,6 +1173,8 @@ fn default_use_seeds(data_type: &DataType) -> bool { // Fixed-width types wider than 8 bytes. DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => true, DataType::FixedSizeBinary(n) => *n > 8, + // Nested types (FSL, List, Struct, Map, …): typically wide. + _ if data_type.is_nested() => true, _ => false, } } @@ -1175,12 +1228,6 @@ impl BasicTrainer for ZoneMapIndexPlugin { params: &str, field: &Field, ) -> Result> { - if field.data_type().is_nested() { - return Err(Error::invalid_input_source( - "A zone map index can only be created on a non-nested field.".into(), - )); - } - let mut params = serde_json::from_str::(params)?; // Resolve None → type-based default so train_index always sees Some(bool). if params.use_seeds.is_none() { @@ -1278,9 +1325,6 @@ impl ScalarIndexPlugin for ZoneMapIndexPlugin { data_type: &DataType, index_details: &prost_types::Any, ) -> Result>> { - if data_type.is_nested() { - return Ok(None); - } let details = index_details.to_msg::().ok(); let Some(rows_per_zone) = details.as_ref().and_then(|d| d.rows_per_zone) else { return Ok(None); @@ -3608,4 +3652,434 @@ mod tests { let result = writer.finish().unwrap(); assert!(result.is_none(), "empty fragment should return None"); } + + // ───────────────────────────── Nested type zone map tests ───────────────────────────── + + use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array}; + + /// Build a FixedSizeList zone map from `rows` (each row is a + /// `Vec>` of length `list_size`), then load and return the index. + async fn train_and_load_fsl(rows: Vec>>, list_size: i32) -> Arc { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), list_size); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + let mut fsl_builder = + arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), list_size); + for row in &rows { + assert_eq!(row.len(), list_size as usize); + for &v in row { + match v { + Some(f) => fsl_builder.values().append_value(f), + None => fsl_builder.values().append_null(), + } + } + fsl_builder.append(true); + } + let fsl_arr: ArrayRef = Arc::new(fsl_builder.finish()); + let n = rows.len() as u64; + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..n)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("failed to load ZoneMapIndex") + } + + use arrow_array::builder::Float32Builder; + use datafusion_common::ScalarValue as SV; + + fn fsl_scalar(values: Vec>) -> SV { + let list_size = values.len() as i32; + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let items: ArrayRef = Arc::new(Float32Array::from(values)); + let arr = Arc::new(FixedSizeListArray::new(item_field, list_size, items, None)); + SV::FixedSizeList(arr) + } + + // Zones for nested types have null min/max; all non-null queries are conservative (not pruned) + // unless every row in the zone is null. + #[tokio::test] + async fn test_fsl_zonemap_conservative_equals() { + // Two zones, 4 rows each, no null rows. + let index = train_and_load_fsl( + vec![ + vec![Some(1.0), Some(2.0)], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)], + vec![Some(7.0), Some(8.0)], + vec![Some(10.0), Some(20.0)], + vec![Some(30.0), Some(40.0)], + vec![Some(50.0), Some(60.0)], + vec![Some(70.0), Some(80.0)], + ], + 2, + ) + .await; + assert_eq!(index.zones.len(), 2); + // min/max are null for nested types + assert!(index.zones[0].min.is_null()); + assert!(index.zones[0].max.is_null()); + assert_eq!(index.zones[0].null_count, 0); + assert_eq!(index.zones[0].nan_count, 0); + + // Non-null Equals: conservative — neither zone is pruned + let q = SargableQuery::Equals(fsl_scalar(vec![Some(100.0), Some(200.0)])); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "non-null Equals on nested type is conservative" + ); + assert!( + index + .evaluate_zone_against_query(&index.zones[1], &q) + .unwrap(), + "non-null Equals on nested type is conservative" + ); + + // IsNull: no null rows → both zones pruned + let q_null = SargableQuery::IsNull(); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q_null) + .unwrap(), + "IsNull pruned when null_count=0" + ); + assert!( + !index + .evaluate_zone_against_query(&index.zones[1], &q_null) + .unwrap(), + "IsNull pruned when null_count=0" + ); + } + + #[tokio::test] + async fn test_fsl_zonemap_null_list() { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), 2); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // Build FSL array with some null entries + let mut builder = arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), 2); + // Row 0: null list + builder.values().append_value(0.0); + builder.values().append_value(0.0); + builder.append(false); + // Row 1: [3.0, 4.0] + builder.values().append_value(3.0); + builder.values().append_value(4.0); + builder.append(true); + let fsl_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..2)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 1, "one null list"); + + // IsNull should match + let q = SargableQuery::IsNull(); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap() + ); + + // Equals null should match (null_count > 0) + let null_scalar = SV::FixedSizeList(Arc::new(FixedSizeListArray::new_null( + item_field.clone(), + 2, + 1, + ))); + let q2 = SargableQuery::Equals(null_scalar); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q2) + .unwrap() + ); + } + + #[tokio::test] + async fn test_fsl_zonemap_is_in_conservative() { + // Two zones (4 rows each), no null rows. + let index = train_and_load_fsl( + vec![ + vec![Some(1.0), Some(2.0)], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)], + vec![Some(7.0), Some(8.0)], + vec![Some(10.0), Some(20.0)], + vec![Some(30.0), Some(40.0)], + vec![Some(50.0), Some(60.0)], + vec![Some(70.0), Some(80.0)], + ], + 2, + ) + .await; + assert_eq!(index.zones.len(), 2); + + // IsIn with non-null values: conservative — neither zone is pruned + let q = SargableQuery::IsIn(vec![ + fsl_scalar(vec![Some(4.0), Some(5.0)]), + fsl_scalar(vec![Some(100.0), Some(200.0)]), + ]); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "IsIn with non-null values is conservative for nested types" + ); + assert!( + index + .evaluate_zone_against_query(&index.zones[1], &q) + .unwrap(), + "IsIn with non-null values is conservative for nested types" + ); + } + + #[tokio::test] + async fn test_fsl_zonemap_value_range_is_none() { + let index = train_and_load_fsl( + vec![vec![Some(1.0), Some(2.0)], vec![Some(3.0), Some(4.0)]], + 2, + ) + .await; + assert_eq!(index.value_range(), None, "FSL index has no scalar range"); + } + + #[tokio::test] + async fn test_fsl_zonemap_all_null_zone() { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), 2); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let mut builder = arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), 2); + for _ in 0..2 { + builder.values().append_value(0.0); + builder.values().append_value(0.0); + builder.append(false); // null list + } + let fsl_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..2)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones[0].null_count, 2); + assert!(index.zones[0].min.is_null(), "all-null zone: min is null"); + + // When every row is null (null_count == zone length), a non-null query is pruned. + let q = SargableQuery::Equals(fsl_scalar(vec![Some(1.0), Some(2.0)])); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "all-null zone (null_count == length) is pruned for non-null target" + ); + + // Range and IsIn are also pruned when all rows are null. + let q_range = SargableQuery::Range(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q_range) + .unwrap(), + "all-null zone is pruned for Range query" + ); + } + + /// Build a zone map on a Struct column and verify training succeeds and stats are correct. + #[tokio::test] + async fn test_struct_zonemap_null_tracking() { + use arrow_array::StructArray; + use arrow_schema::Fields; + + let item_field = Arc::new(Field::new("x", DataType::Int32, true)); + let struct_type = DataType::Struct(Fields::from(vec![item_field.clone()])); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, struct_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // 3 non-null struct rows + let x_values: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![10, 20, 30])); + let struct_arr: ArrayRef = + Arc::new(StructArray::from(vec![(item_field.clone(), x_values)])); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..3)); + let batch = RecordBatch::try_new(schema.clone(), vec![struct_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(10)), + ) + .await + .unwrap(); + + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 0, "no null struct rows"); + assert_eq!(index.zones[0].nan_count, 0); + // min/max are typed null ScalarValues (exact representation varies by DataFusion version) + // but IsNull correctly returns false and Equals is conservative + + // IsNull: no null rows → zone is pruned + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &SargableQuery::IsNull()) + .unwrap() + ); + + // Non-null Equals: conservative (not pruned) since not all rows are null + assert!( + index + .evaluate_zone_against_query( + &index.zones[0], + &SargableQuery::Equals(SV::Int32(Some(99))) + ) + .unwrap() + ); + + // value_range: always None for nested types + assert_eq!(index.value_range(), None); + } + + /// Build a zone map on a List column and verify null tracking works. + #[tokio::test] + async fn test_list_zonemap_null_tracking() { + use arrow_array::builder::ListBuilder; + + let item_field = Arc::new(Field::new("item", DataType::Int32, true)); + let list_type = DataType::List(item_field.clone()); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, list_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // Build: [null, [1,2], [3]] + let mut builder = ListBuilder::new(arrow_array::builder::Int32Builder::new()); + builder.append_null(); + builder.values().append_value(1); + builder.values().append_value(2); + builder.append(true); + builder.values().append_value(3); + builder.append(true); + let list_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..3)); + let batch = RecordBatch::try_new(schema.clone(), vec![list_arr, row_addr]).unwrap(); + + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(10)), + ) + .await + .unwrap(); + + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 1, "one null list row"); + // min/max are typed null ScalarValues for nested types + + // IsNull search: the null bitmap is populated during training so + // the result is an exact set containing just the null row (row address 0). + let result = index + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + let mut exact_nulls = RowAddrTreeMap::new(); + exact_nulls.insert(0); // only row 0 is null + assert_eq!(result, SearchResult::exact(exact_nulls)); + } } diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index ba87efdff6c..adea488689b 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -2868,12 +2868,6 @@ impl DatasetIndexInternalExt for Dataset { // so the optimizer treats coverage as unknown. let mut fragment_bitmaps: HashMap<(String, String), Option> = HashMap::new(); for index in indices.iter().filter(|idx| { - let idx_schema = schema.project_by_ids(idx.fields.as_slice(), true); - let is_vector_index = idx_schema - .fields - .iter() - .any(|f| is_vector_field(f.data_type())); - // Check if this is an FTS index by looking at index details let is_fts_index = if let Some(details) = &idx.index_details { IndexDetails(details.clone()).supports_fts() @@ -2887,7 +2881,7 @@ impl DatasetIndexInternalExt for Dataset { !bitmap.is_empty() && !(bitmap & self.fragment_bitmap.as_ref()).is_empty() }); - idx.fields.len() == 1 && !is_vector_index && (has_non_empty_bitmap || is_fts_index) + idx.fields.len() == 1 && (has_non_empty_bitmap || is_fts_index) }) { let field = index.fields[0]; let field = schema.field_by_id(field).ok_or_else(|| { diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs index ef01a2e3b10..16bb655a871 100644 --- a/rust/lance/src/index/create.rs +++ b/rust/lance/src/index/create.rs @@ -21,7 +21,9 @@ use crate::{ use futures::{FutureExt, future::BoxFuture}; use lance_core::datatypes::format_field_path; use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress}; -use lance_index::{IndexParams, IndexType, scalar::CreatedIndex}; +use lance_index::{ + IndexParams, IndexType, registry::plugin_name_from_details_url, scalar::CreatedIndex, +}; use lance_index::{ metrics::NoOpMetricsCollector, scalar::{ @@ -268,7 +270,10 @@ impl<'a> CreateIndexBuilder<'a> { let index_name = if let Some(name) = self.name.take() { name } else { - // Generate default name with collision handling + // Generate default name with collision handling. + // A name is available when there is no existing index with: + // - the same name AND different fields, OR + // - the same name AND same field BUT a different index kind let column_path = resolved_fts_field .as_ref() .map(|resolved| { @@ -282,11 +287,11 @@ impl<'a> CreateIndexBuilder<'a> { let base_name = format!("{column_path}_idx"); let mut candidate = base_name.clone(); let mut counter = 2; // Start with no suffix, then use _2, _3, ... - // Find unique name by appending numeric suffix if needed - while indices - .iter() - .any(|idx| idx.name == candidate && idx.fields != [field.id]) - { + while indices.iter().any(|idx| { + idx.name == candidate + && (idx.fields != [field.id] + || !index_matches_type(idx, self.index_type, self.params)) + }) { candidate = format!("{base_name}_{counter}"); counter += 1; } @@ -841,8 +846,6 @@ impl<'a> CreateIndexBuilder<'a> { build_index_metadata_from_segments(self.dataset, &index_name, field.id, segments) .await?; - // Collect all same-name indices for removal when replace is set, - // matching the standard execute() path behavior. let removed_indices = if self.replace { existing_named_indices .into_iter() @@ -876,6 +879,31 @@ impl<'a> CreateIndexBuilder<'a> { } } +/// Returns true if an existing `IndexMetadata` matches the given type +fn index_matches_type( + idx: &IndexMetadata, + index_type: IndexType, + params: &dyn IndexParams, +) -> bool { + let Some(d) = &idx.index_details else { + // Fallback for legacy indexes, assume we are not trying to change the type + return true; + }; + // When index_type is Scalar the actual type is carried in ScalarIndexParams as a + // plugin name string (e.g. "zonemap"). The registry uses the same normalization + // for type_url lookup: lowercase the last path segment and strip "indexdetails". + // Compare directly instead of going through IndexType so this path stays valid + // as we move away from the IndexType enum. + if index_type == IndexType::Scalar + && let Some(scalar_params) = params.as_any().downcast_ref::() + { + return scalar_params.index_type.to_lowercase() + == plugin_name_from_details_url(&d.type_url); + } + + index_type.matches_details(d) +} + fn is_builtin_vector_index(index_type: IndexType, params: &dyn IndexParams) -> bool { params.index_name() == LANCE_VECTOR_INDEX && matches!(