feat(index): add zone map support for all data types

python/python/lance/dataset.py

@@ -3292,7 +3292,7 @@ def _prepare_scalar_index_request(
             if hasattr(field_type, "storage_type"):
                 field_type = field_type.storage_type
 
-            if index_type in ["BTREE", "BITMAP", "ZONEMAP"]:
+            if index_type in ["BTREE", "BITMAP"]:
                 if (
                     not pa.types.is_integer(field_type)
                     and not pa.types.is_floating(field_type)
@@ -3303,7 +3303,7 @@ def _prepare_scalar_index_request(
                     and not pa.types.is_fixed_size_binary(field_type)
                 ):
                     raise TypeError(
-                        f"BTREE/BITMAP/ZONEMAP index column {column} must be int",
+                        f"BTREE/BITMAP index column {column} must be int",
                         ", float, bool, str, large_str, fixed-size-binary, or temporal",
                     )
             elif index_type == "LABEL_LIST":

python/python/tests/test_scalar_index.py

@@ -2860,6 +2860,54 @@ def test_zonemap_index_remapping(tmp_path: Path):
     assert result.num_rows == 501  # 1000..1500 inclusive
 
 
+def test_zonemap_fsl_column(tmp_path: Path):
+    """Zone map can be created on a FixedSizeList column and accelerates IS NULL."""
+    dim = 8
+    n = 1000
+    rng = np.random.default_rng(42)
+    vectors = rng.standard_normal((n, dim)).astype(np.float32)
+    vec_type = pa.list_(pa.float32(), dim)
+    # Every 10th row is null
+    vec_list = [None if i % 10 == 0 else v.tolist() for i, v in enumerate(vectors)]
+    tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))})
+    ds = lance.write_dataset(tbl, tmp_path)
+    ds.create_scalar_index("vec", index_type="ZONEMAP")
+
+    scanner = ds.scanner(filter="vec IS NULL", prefilter=True)
+    plan = scanner.explain_plan()
+    assert "ScalarIndexQuery" in plan
+    result = scanner.to_table()
+    assert result.num_rows == 100  # every 10th row is null
+
+
+def test_vector_and_zonemap_on_fsl_column(tmp_path: Path):
+    """Vector index and zone map can coexist on the same FSL column."""
+    dim = 16
+    n = 2000
+    rng = np.random.default_rng(0)
+    vectors = rng.standard_normal((n, dim)).astype(np.float32)
+    vec_type = pa.list_(pa.float32(), dim)
+    # Every 20th row is null
+    vec_list = [None if i % 20 == 0 else v.tolist() for i, v in enumerate(vectors)]
+    tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))})
+    ds = lance.write_dataset(tbl, tmp_path)
+
+    ds.create_index("vec", index_type="IVF_PQ", num_partitions=4, num_sub_vectors=2)
+    ds.create_scalar_index("vec", index_type="ZONEMAP")
+
+    # Vector search still works
+    query = vectors[5]
+    result = ds.scanner(nearest={"column": "vec", "q": query, "k": 10}).to_table()
+    assert result.num_rows == 10
+
+    # IS NULL is zone-map-accelerated
+    scanner = ds.scanner(filter="vec IS NULL", prefilter=True)
+    plan = scanner.explain_plan()
+    assert "ScalarIndexQuery" in plan
+    null_result = scanner.to_table()
+    assert null_result.num_rows == 100  # every 20th row is null
+
+
 def test_bloomfilter_index(tmp_path: Path):
     """Test create bloomfilter index"""
     tbl = pa.Table.from_arrays([pa.array([i for i in range(10000)])], names=["values"])

rust/lance-index-core/src/lib.rs

@@ -279,6 +279,31 @@ impl IndexType {
         .max()
         .unwrap_or(1)
     }
+
+    pub fn matches_details(&self, details: &prost_types::Any) -> bool {
+        let url = &details.type_url;
+        match self {
+            Self::Scalar | Self::BTree => url.ends_with("BTreeIndexDetails"),
+            Self::Bitmap => url.ends_with("BitmapIndexDetails"),
+            Self::LabelList => url.ends_with("LabelListIndexDetails"),
+            Self::Inverted => url.ends_with("InvertedIndexDetails"),
+            Self::NGram => url.ends_with("NGramIndexDetails"),
+            Self::ZoneMap => url.ends_with("ZoneMapIndexDetails"),
+            Self::BloomFilter => url.ends_with("BloomFilterIndexDetails"),
+            Self::RTree => url.ends_with("RTreeIndexDetails"),
+            Self::Fm => url.ends_with("FMIndexDetails"),
+            Self::FragmentReuse => url.ends_with("FragmentReuseIndexDetails"),
+            Self::MemWal => url.ends_with("MemWalIndexDetails"),
+            Self::Vector
+            | Self::IvfFlat
+            | Self::IvfSq
+            | Self::IvfPq
+            | Self::IvfHnswSq
+            | Self::IvfHnswPq
+            | Self::IvfHnswFlat
+            | Self::IvfRq => url.ends_with("VectorIndexDetails"),
+        }
+    }
 }
 
 pub trait IndexParams: Send + Sync {

rust/lance-index/src/registry.rs

@@ -16,6 +16,21 @@ use crate::{
     },
 };
 
+/// Derive the scalar index plugin name from a details type URL.
+///
+/// Takes the last `.`-separated segment, lowercases it, and strips any trailing
+/// `"indexdetails"` suffix so the result matches the plugin name used in
+/// [`IndexPluginRegistry`]. For example, `/lance.index.pb.ZoneMapIndexDetails`
+/// yields `"zonemap"`.
+pub fn plugin_name_from_details_url(type_url: &str) -> String {
+    let segment = type_url.split('.').next_back().unwrap_or(type_url);
+    let lower = segment.to_lowercase();
+    lower
+        .strip_suffix("indexdetails")
+        .map(|s| s.to_string())
+        .unwrap_or(lower)
+}
+
 /// Derive a human-readable index type name from a details type URL.
 ///
 /// The display name is the final `.`-separated segment of the type URL with any
@@ -42,12 +57,7 @@ impl IndexPluginRegistry {
     }
 
     fn get_plugin_name_from_details_name(&self, details_name: &str) -> String {
-        let details_name = Self::normalize_plugin_name(details_name);
-        if details_name.ends_with("indexdetails") {
-            details_name.replace("indexdetails", "")
-        } else {
-            details_name
-        }
+        plugin_name_from_details_url(details_name)
     }
 
     /// Adds a plugin to the registry, using the name of the details message to determine