diff --git a/docs/src/distributed-indexing.md b/docs/src/distributed-indexing.md index ba54e9bd..6354539c 100755 --- a/docs/src/distributed-indexing.md +++ b/docs/src/distributed-indexing.md @@ -270,10 +270,7 @@ dataset = lance.dataset("path/to/dataset") # Build distributed index updated_dataset = lr.create_scalar_index( - uri=dataset.uri, - column="text", - index_type="INVERTED", - num_workers=4 + uri=dataset.uri, column="text", index_type="INVERTED", num_workers=4 ) # Verify index creation @@ -282,8 +279,7 @@ print(f"Index list: {indices}") # Use index for search results = updated_dataset.scanner( - full_text_query="search term", - columns=["id", "text"] + full_text_query="search term", columns=["id", "text"] ).to_table() print(f"Search results: {results}") ``` @@ -304,7 +300,9 @@ updated_dataset = lr.create_scalar_index( # Example queries updated_dataset.scanner(filter="id = 100", columns=["id", "text"]).to_table() -updated_dataset.scanner(filter="id >= 200 AND id < 800", columns=["id", "text"]).to_table() +updated_dataset.scanner( + filter="id >= 200 AND id < 800", columns=["id", "text"] +).to_table() ``` ### Vector Index (IVF_PQ / IVF_RQ / IVF_SQ / IVF_FLAT) @@ -321,7 +319,7 @@ updated_dataset = lr.create_index( num_partitions=256, num_sub_vectors=16, sample_rate=64, - metric="l2" + metric="l2", ) # Build a distributed IVF_SQ index @@ -399,11 +397,11 @@ print(plan) ```python updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - num_workers=4, - ray_remote_args={"num_cpus": 2, "resources": {"custom_resource": 1}} + uri="path/to/dataset", + column="text", + index_type="INVERTED", + num_workers=4, + ray_remote_args={"num_cpus": 2, "resources": {"custom_resource": 1}}, ) ``` @@ -444,21 +442,21 @@ The current global Pool integration is limited to `vector_search()`. The same p ```python # Create index with custom name updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - name="my_text_index", - num_workers=4 + uri="path/to/dataset", + column="text", + index_type="INVERTED", + name="my_text_index", + num_workers=4, ) # Try to create another index with the same name (will replace by default) updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - name="my_text_index", # Same name as before - replace=True, # Explicitly allow replacement (default behavior) - num_workers=4 + uri="path/to/dataset", + column="text", + index_type="INVERTED", + name="my_text_index", # Same name as before + replace=True, # Explicitly allow replacement (default behavior) + num_workers=4, ) # Prevent index replacement @@ -466,12 +464,12 @@ import lance_ray as lr try: updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - name="my_text_index", # Same name as existing index - replace=False, # Prevent replacement - num_workers=4 + uri="path/to/dataset", + column="text", + index_type="INVERTED", + name="my_text_index", # Same name as existing index + replace=False, # Prevent replacement + num_workers=4, ) except ValueError as e: print(f"Index creation failed: {e}") diff --git a/docs/src/examples.md b/docs/src/examples.md index 39164a35..47f2331f 100644 --- a/docs/src/examples.md +++ b/docs/src/examples.md @@ -37,9 +37,7 @@ print(f"Filtered count: {filtered_ds.count()}") # Read with column selection and filtering ds_filtered = read_lance( - "sample_dataset.lance", - columns=["user_id", "name", "score"], - filter="score > 75.0" + "sample_dataset.lance", columns=["user_id", "name", "score"], filter="score > 75.0" ) print(f"Schema: {ds_filtered.schema()}") ``` @@ -51,16 +49,14 @@ print(f"Schema: {ds_filtered.schema()}") from lance_ray import add_columns import pyarrow as pa + def add_computed_column(batch: pa.RecordBatch) -> pa.RecordBatch: df = batch.to_pandas() - df['computed'] = df['value'] * 2 + df['id'] + df["computed"] = df["value"] * 2 + df["id"] return pa.RecordBatch.from_pandas(df[["computed"]]) -add_columns( - uri="sample_dataset.lance", - transform=add_computed_column, - concurrency=4 -) + +add_columns(uri="sample_dataset.lance", transform=add_computed_column, concurrency=4) ``` ## Using Namespace @@ -110,7 +106,7 @@ from lance_ray import read_lance, write_lance # Initialize Ray ray.init() -# Connect to AWS Glue catalog +# Connect to AWS Glue catalog # using the default account and region in the current AWS environment namespace = ln.connect("glue", {}) @@ -119,10 +115,10 @@ data = ray.data.range(1000).map(lambda row: {"id": row["id"], "value": row["id"] # Write to Lance format using metadata catalog write_lance( - data, - uri="s3://my-bucket/my-table", - namespace=namespace, - table_id=["default", "my_table"] + data, + uri="s3://my-bucket/my-table", + namespace=namespace, + table_id=["default", "my_table"], ) # Read Lance dataset back using metadata catalog diff --git a/lance_ray/compaction.py b/lance_ray/compaction.py index 9b0f1252..f7018748 100644 --- a/lance_ray/compaction.py +++ b/lance_ray/compaction.py @@ -140,7 +140,9 @@ def compact_files( # Step 1: Create the compaction plan # Compaction.plan requires a dict; CompactionOptions is a TypedDict, so # an empty instance stands in for "all defaults" when the caller omits it. - compaction_plan = Compaction.plan(dataset, compaction_options or CompactionOptions()) + compaction_plan = Compaction.plan( + dataset, compaction_options or CompactionOptions() + ) logger.info(f"Compaction plan created with {compaction_plan.num_tasks()} tasks") diff --git a/lance_ray/index.py b/lance_ray/index.py index ea01bf33..88bc4683 100755 --- a/lance_ray/index.py +++ b/lance_ray/index.py @@ -105,9 +105,7 @@ def _distribute_fragments_balanced( # Greedy assignment: assign each fragment to the segment with minimum workload for frag_info in fragment_info: - min_workload_idx = min( - range(num_segments), key=lambda i: segment_workloads[i] - ) + min_workload_idx = min(range(num_segments), key=lambda i: segment_workloads[i]) segment_batches[min_workload_idx].append(frag_info["id"]) segment_workloads[min_workload_idx] += frag_info["size"] diff --git a/lance_ray/utils.py b/lance_ray/utils.py index 12fe3659..5e3c8b75 100644 --- a/lance_ray/utils.py +++ b/lance_ray/utils.py @@ -238,9 +238,7 @@ def resolve_namespace_table( ) location = describe_response.location if location is None: - raise ValueError( - "Namespace did not return a 'location' for the table" - ) + raise ValueError("Namespace did not return a 'location' for the table") if describe_response.storage_options: merged_storage_options.update(describe_response.storage_options) return location, merged_storage_options diff --git a/pyproject.toml b/pyproject.toml index b7d14df8..77b457c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "ray[data]>=2.41.0", - "pylance==9.0.0rc1", + "pylance>=10.0.0b3", "lance-namespace", "packaging", "pyarrow>=17.0.0", diff --git a/uv.lock b/uv.lock index 42589210..2a0048df 100644 --- a/uv.lock +++ b/uv.lock @@ -221,7 +221,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -387,7 +387,7 @@ requires-dist = [ { name = "more-itertools", marker = "python_full_version < '3.12'", specifier = ">=2.6.0" }, { name = "packaging" }, { name = "pyarrow", specifier = ">=17.0.0" }, - { name = "pylance", specifier = "==9.0.0rc1" }, + { name = "pylance", specifier = ">=10.0.0b3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, @@ -1024,7 +1024,7 @@ wheels = [ [[package]] name = "pylance" -version = "9.0.0rc1" +version = "10.0.0b3" source = { registry = "https://pypi.fury.io/lance-format" } dependencies = [ { name = "lance-namespace" }, @@ -1033,12 +1033,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_vEHBE/pylance-9.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0b6b02a1808bb3072ee7fe4e36614cae6f86302513e73ec7f55b2234a963b24" }, - { url = "https://pypi.fury.io/lance-format/-/ver_1Jipm4/pylance-9.0.0rc1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30f0ebf0d88034301819eb964f9236ce555aaa58e7ab89c5975a3e2250bbb405" }, - { url = "https://pypi.fury.io/lance-format/-/ver_IvKxo/pylance-9.0.0rc1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44609ea2615ea6e684b85478d1694af2026458f61cf7895ecc75e238bfd17aa8" }, - { url = "https://pypi.fury.io/lance-format/-/ver_2hidj1/pylance-9.0.0rc1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:182167a8dba9eeabffbffd53bd5b8548613d4d459b7cd7b34a840dd00cbb806f" }, - { url = "https://pypi.fury.io/lance-format/-/ver_1dFx3r/pylance-9.0.0rc1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8a63b11e814b7eab758bcaf0d6f97eb05ea86203d9fb0af718c462c24c7d6c9c" }, - { url = "https://pypi.fury.io/lance-format/-/ver_2a8dSh/pylance-9.0.0rc1-cp310-abi3-win_amd64.whl", hash = "sha256:2ff8b953ae2b0550490c1a7efd210aa91bc223d200ffac28849056cfd7436d97" }, + { url = "https://pypi.fury.io/lance-format/-/ver_2c81m8/pylance-10.0.0b3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:acb0cbe166a3e96a35cb19c6fff723e6e15e913f3da958ae8bd16c5fbfca4519" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1eJFKW/pylance-10.0.0b3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:069d911c35a844e9c895990d65b8417f2848d82862f51cbec3ae9466890803e8" }, + { url = "https://pypi.fury.io/lance-format/-/ver_zirfN/pylance-10.0.0b3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:342615ca5ce7fffb92f38f62fa93ebdd4e630e3e09226a8dc36dd09aee2437e9" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1MbmzH/pylance-10.0.0b3-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:37feb4c5b4d390a2617ff5472a2c227910364c347f0004ba5de537eca440d6bd" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1MQImE/pylance-10.0.0b3-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4009c733254e12555b42b6b5186677017eee377f864dd7253f8596c8a0b80135" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1CrJT7/pylance-10.0.0b3-cp310-abi3-win_amd64.whl", hash = "sha256:2eb233944449427a3d14b2034c256c5bf2a59eef120fb43929467c46b78a6a4a" }, ] [[package]]