diff --git a/modules/backend_maturin/private/tools/generate_cargo_lock.py b/modules/backend_maturin/private/tools/generate_cargo_lock.py index 044772e4..93a4f3f8 100644 --- a/modules/backend_maturin/private/tools/generate_cargo_lock.py +++ b/modules/backend_maturin/private/tools/generate_cargo_lock.py @@ -24,6 +24,33 @@ def derive_default_output(sdist_path: Path) -> str: return "Cargo.lock" +def _find_cargo_toml_dir_from_pyproject(sdist_root: Path) -> Path | None: + pyproject_path = sdist_root / "pyproject.toml" + if not pyproject_path.exists(): + return sdist_root + + try: + import tomllib + except ImportError: + try: + from pip._vendor import tomli as tomllib # type: ignore[no-redef] + except ImportError: + print("Warning: tomllib/tomli not found. Searching for Cargo.toml instead.", file=sys.stderr) + return None + + try: + with open(pyproject_path, "rb") as f: + pyproject_data = tomllib.load(f) + manifest_path = pyproject_data.get("tool", {}).get("maturin", {}).get("manifest-path", "Cargo.toml") + return (sdist_root / manifest_path).parent + except Exception as e: + print( + f"Warning: Failed to parse pyproject.toml: {e}. Searching for Cargo.toml instead.", + file=sys.stderr, + ) + return None + + def main(): parser = argparse.ArgumentParser(description="Generate Cargo.lock for an sdist.") parser.add_argument("--sdist", required=True, help="Path to the sdist tarball.") @@ -64,34 +91,7 @@ def main(): # Find Cargo.toml. # We also want to support manifest-path if defined in pyproject.toml - cargo_toml_dir = sdist_root - pyproject_path = sdist_root / "pyproject.toml" - if pyproject_path.exists(): - try: - import tomllib - except ImportError: - try: - from pip._vendor import tomli as tomllib # type: ignore[no-redef] - except ImportError: - tomllib = None # type: ignore[assignment] - - if tomllib: - with open(pyproject_path, "rb") as f: - try: - pyproject_data = tomllib.load(f) - manifest_path = ( - pyproject_data.get("tool", {}).get("maturin", {}).get("manifest-path", "Cargo.toml") - ) - cargo_toml_dir = (sdist_root / manifest_path).parent - except Exception as e: - print( - f"Warning: Failed to parse pyproject.toml: {e}. Searching for Cargo.toml instead.", - file=sys.stderr, - ) - cargo_toml_dir = None - else: - print("Warning: tomllib/tomli not found. Searching for Cargo.toml instead.", file=sys.stderr) - cargo_toml_dir = None + cargo_toml_dir = _find_cargo_toml_dir_from_pyproject(sdist_root) if not cargo_toml_dir or not (cargo_toml_dir / "Cargo.toml").exists(): # Fallback to searching diff --git a/modules/backend_maturin/private/tools/rust_common.py b/modules/backend_maturin/private/tools/rust_common.py index ecd249d3..2830b2dd 100644 --- a/modules/backend_maturin/private/tools/rust_common.py +++ b/modules/backend_maturin/private/tools/rust_common.py @@ -41,6 +41,15 @@ def get_pyo3_version(cargo_dir: Path): return None +def _symlink_stdlib(src_path: Path, dst_path: Path, label: str) -> None: + if not src_path.exists(): + raise RuntimeError(f"{label} missing at {src_path}") + if dst_path.exists() or dst_path.is_symlink(): + dst_path.unlink(missing_ok=True) + dst_path.parent.mkdir(parents=True, exist_ok=True) + os.symlink(src_path.resolve(), dst_path) + + def configure_rust_env(ctx, cargo_dir: Path, is_maturin: bool = False): rust_config = None for layer_config in load_layers(ctx): @@ -198,24 +207,12 @@ def configure_rust_env(ctx, cargo_dir: Path, is_maturin: bool = False): target_stdlib_dst_path = sysroot_dir / Path(f"lib/rustlib/{target_triple}/lib") try: - if host_stdlib_src_path: - if not host_stdlib_src_path.exists(): - raise RuntimeError(f"host_stdlib missing at {host_stdlib_src_path}") - if host_stdlib_dst_path.exists() or host_stdlib_dst_path.is_symlink(): - host_stdlib_dst_path.unlink(missing_ok=True) - - host_stdlib_dst_path.parent.mkdir(parents=True, exist_ok=True) - os.symlink(host_stdlib_src_path.resolve(), host_stdlib_dst_path) + if host_stdlib_src_path and host_stdlib_dst_path: + _symlink_stdlib(host_stdlib_src_path, host_stdlib_dst_path, "host_stdlib") target_stdlib_src_path = ctx.prefix / Path(f"external/{cross_repo_name}/lib/rustlib/{target_triple}/lib") - if target_stdlib_src_path and target_stdlib_dst_path: - if not target_stdlib_src_path.exists(): - raise RuntimeError(f"target_stdlib missing at {target_stdlib_src_path}") - if target_stdlib_src_path != host_stdlib_src_path: - if target_stdlib_dst_path.exists() or target_stdlib_dst_path.is_symlink(): - target_stdlib_dst_path.unlink(missing_ok=True) - target_stdlib_dst_path.parent.mkdir(parents=True, exist_ok=True) - os.symlink(target_stdlib_src_path.resolve(), target_stdlib_dst_path) + if target_stdlib_src_path != host_stdlib_src_path: + _symlink_stdlib(target_stdlib_src_path, target_stdlib_dst_path, "target_stdlib") except Exception as e: raise RuntimeError(f"Failed to populate hermetic Rust sysroot: {e}") from e diff --git a/pycross/private/backends.bzl b/pycross/private/backends.bzl index 85c4a717..8c49a360 100644 --- a/pycross/private/backends.bzl +++ b/pycross/private/backends.bzl @@ -63,25 +63,24 @@ def _collect_registrations(modules, warn = _print_warn): for pyproject_backend in tag.pyproject_backends: if pyproject_backend in backend_to_rule and not module.is_root: - if pyproject_backend in root_pyproject_backends: - continue - warn( - "Ignoring duplicate pyproject backend '{}' -> '{}' from module '{}' (already mapped to '{}')".format( - pyproject_backend, - name, - module.name, - backend_to_rule[pyproject_backend], - ), - ) - else: - backend_to_rule[pyproject_backend] = name - if module.is_root: - root_pyproject_backends[pyproject_backend] = True - - if tag.default: - if default_backend and not module.is_root: - if root_default: - continue + if pyproject_backend not in root_pyproject_backends: + warn( + "Ignoring duplicate pyproject backend '{}' -> '{}' from module '{}' (already mapped to '{}')".format( + pyproject_backend, + name, + module.name, + backend_to_rule[pyproject_backend], + ), + ) + continue + backend_to_rule[pyproject_backend] = name + if module.is_root: + root_pyproject_backends[pyproject_backend] = True + + if not tag.default: + continue + if default_backend and not module.is_root: + if not root_default: warn( "Ignoring default backend '{}' from module '{}' (already set to '{}' by module '{}')".format( name, @@ -90,10 +89,10 @@ def _collect_registrations(modules, warn = _print_warn): default_backend_module, ), ) - else: - default_backend = name - default_backend_module = module.name - root_default = module.is_root + continue + default_backend = name + default_backend_module = module.name + root_default = module.is_root if not default_backend: fail("No default build backend registered. Set `default = True` on one `backends.register` tag.") diff --git a/pycross/private/build/rules/common_attrs.bzl b/pycross/private/build/rules/common_attrs.bzl index a22df773..3917e55b 100644 --- a/pycross/private/build/rules/common_attrs.bzl +++ b/pycross/private/build/rules/common_attrs.bzl @@ -35,19 +35,19 @@ def resolve_path_tools(ctx): file = info.executable, files_to_run = target[DefaultInfo].files_to_run, )) - else: - exe = target[DefaultInfo].files_to_run.executable - if not exe: - files = target[DefaultInfo].files.to_list() - if files: - exe = files[0] - if not exe: - fail("Tool target must provide an executable: " + str(target.label)) - result.append(struct( - name = exe.basename, - file = exe, - files_to_run = target[DefaultInfo].files_to_run, - )) + continue + + exe = target[DefaultInfo].files_to_run.executable + if not exe: + files = target[DefaultInfo].files.to_list() + exe = files[0] if files else None + if not exe: + fail("Tool target must provide an executable: " + str(target.label)) + result.append(struct( + name = exe.basename, + file = exe, + files_to_run = target[DefaultInfo].files_to_run, + )) return result COMMON_BUILD_ATTRS = { @@ -188,11 +188,9 @@ def group_tool_deps(tool_deps_list): """ result = {} for dep in tool_deps_list: - if PycrossPackageInfo in dep: - name = dep[PycrossPackageInfo].package_name - if name not in result: - result[name] = [] - result[name].append(dep) + if PycrossPackageInfo not in dep: + continue + result.setdefault(dep[PycrossPackageInfo].package_name, []).append(dep) return result def get_wheel(target): diff --git a/pycross/private/build/tools/utils/cc_toolchain.py b/pycross/private/build/tools/utils/cc_toolchain.py index 74e9f492..fad04b77 100644 --- a/pycross/private/build/tools/utils/cc_toolchain.py +++ b/pycross/private/build/tools/utils/cc_toolchain.py @@ -32,20 +32,18 @@ def parse_ar_and_guess_ranlib(ar: str | None) -> Tuple[List[str], Path | None]: ar_list[0] = resolved_ar stem = ar_path.stem - if stem == "ar" or stem.endswith(("-ar", "_ar")): - ranlib_stem = stem[:-2] + "ranlib" - ranlib_name = ranlib_stem + ar_path.suffix - - if ar_path.is_absolute(): - guessed_ranlib_path = ar_path.parent / ranlib_name - if guessed_ranlib_path.exists(): - return ar_list, guessed_ranlib_path - else: - resolved_ranlib = shutil.which(ranlib_name) - if resolved_ranlib: - return ar_list, Path(resolved_ranlib) + if not (stem == "ar" or stem.endswith(("-ar", "_ar"))): + return ar_list, None + + ranlib_stem = stem[:-2] + "ranlib" + ranlib_name = ranlib_stem + ar_path.suffix - return ar_list, None + if ar_path.is_absolute(): + guessed_ranlib_path = ar_path.parent / ranlib_name + return ar_list, guessed_ranlib_path if guessed_ranlib_path.exists() else None + else: + resolved_ranlib = shutil.which(ranlib_name) + return ar_list, Path(resolved_ranlib) if resolved_ranlib else None _FILTERED_LINKER_ARGS = ( @@ -81,23 +79,17 @@ def parse_ar_and_guess_ranlib(ar: str | None) -> Tuple[List[str], Path | None]: def get_wrapper_flags(cflags: str) -> List[str]: """Extract target and sysroot flags to forward to compiler wrappers.""" - possible_flags = ["-target", "--target", "--sysroot", "-isysroot", "-mmacosx-version-min"] + possible_flags = ("-target", "--target", "--sysroot", "-isysroot", "-mmacosx-version-min") result = [] split_cflags = shlex.split(cflags) for i, flag in enumerate(split_cflags): - for possible_flag in possible_flags: - if not (flag.startswith(possible_flag)): - continue - if "=" in flag: - flag, value = flag.split("=", 1) - additions = [f"{flag}={value}"] - else: - flag, value = flag, split_cflags[i + 1] - additions = [flag, value] - - if not flag == possible_flag: - continue - result.extend(additions) + flag_name, sep, value = flag.partition("=") + if flag_name not in possible_flags: + continue + if sep: + result.append(f"{flag_name}={value}") + else: + result.extend([flag_name, split_cflags[i + 1]]) return result diff --git a/pycross/private/cycle_dep_needed.bzl b/pycross/private/cycle_dep_needed.bzl index 9ee8fe1f..a2e356fa 100644 --- a/pycross/private/cycle_dep_needed.bzl +++ b/pycross/private/cycle_dep_needed.bzl @@ -26,6 +26,20 @@ load("//pycross/private:util.bzl", "parse_package_key") # ---- BFS reachability ------------------------------------------------------- +def _node_env(markers_env, node_extra): + base_extra = markers_env.get("extra", "") + if type(base_extra) == "list": + if base_extra[0] == node_extra: + return markers_env + env = dict(markers_env) + env["extra"] = [node_extra] + base_extra[1:] + return env + if base_extra == node_extra: + return markers_env + env = dict(markers_env) + env["extra"] = node_extra + return env + def _active_neighbors(edges, node, markers_env): """Returns the list of neighbor nodes reachable from `node` given marker values. @@ -45,18 +59,7 @@ def _active_neighbors(edges, node, markers_env): marker_str = edge.get("marker") if marker_str: if node_env == None: - base_extra = markers_env.get("extra", "") - if type(base_extra) == "list": - if base_extra[0] == node_extra: - node_env = markers_env - else: - node_env = dict(markers_env) - node_env["extra"] = [node_extra] + base_extra[1:] - elif base_extra == node_extra: - node_env = markers_env - else: - node_env = dict(markers_env) - node_env["extra"] = node_extra + node_env = _node_env(markers_env, node_extra) parsed_marker = pypackaging.markers.parse(marker_str) if not pypackaging.markers.evaluate(parsed_marker, node_env): continue diff --git a/pycross/private/cycle_member_marker_deps.bzl b/pycross/private/cycle_member_marker_deps.bzl index 73e27615..1ab80465 100644 --- a/pycross/private/cycle_member_marker_deps.bzl +++ b/pycross/private/cycle_member_marker_deps.bzl @@ -65,12 +65,12 @@ def find_unconditional_deps(member, other_members, edges): dep = edge["dep"] if dep not in cycle_members: continue - has_marker = bool(edge.get("marker")) - if not has_marker and dep not in visited: - visited[dep] = True - if dep != member: - unconditional[dep] = True - queue.append(dep) + if edge.get("marker") or dep in visited: + continue + visited[dep] = True + if dep != member: + unconditional[dep] = True + queue.append(dep) return sorted(unconditional.keys()) @@ -116,10 +116,11 @@ def compute_reachability_groups(member, other_members, edges): collapsible = {} # node -> predecessor for node in other_members: node_inbound = inbound.get(node, []) - if len(node_inbound) == 1: - pred, has_marker = node_inbound[0] - if not has_marker and pred != member and pred in members_set: - collapsible[node] = pred + if len(node_inbound) != 1: + continue + pred, has_marker = node_inbound[0] + if not has_marker and pred != member and pred in members_set: + collapsible[node] = pred # Step 3: resolve chains. If C -> B -> A in the collapsible map, C's # representative is A (the first non-collapsible ancestor). @@ -373,10 +374,10 @@ def pycross_cycle_member_marker_deps( if cg.eval_match_name and native.existing_rule(cg.eval_match_name): match_name = cg.eval_match_name else: - if edges_json == None: - edges_json = json.encode(edges) - if not native.existing_rule(cg.pair_name): + if edges_json == None: + edges_json = json.encode(edges) + # Reachability evaluator: returns FeatureFlagInfo("true"/"false") pycross_cycle_dep_needed( name = cg.pair_name, diff --git a/pycross/private/format_extension.bzl b/pycross/private/format_extension.bzl index ac61329e..343aa540 100644 --- a/pycross/private/format_extension.bzl +++ b/pycross/private/format_extension.bzl @@ -379,13 +379,12 @@ def make_format_extension( for tag in module.tags.package: ws_name = tag.workspace if not ws_name: - if len(workspaces_in_module) == 1: - ws_name = workspaces_in_module[0] - else: + if len(workspaces_in_module) != 1: fail("package '{}' must specify workspace (module defines workspaces: {})".format( tag.name, ", ".join(workspaces_in_module) if workspaces_in_module else "none", )) + ws_name = workspaces_in_module[0] if ws_name not in workspaces_in_module: fail("package '{}': workspace '{}' not declared by this module".format(tag.name, ws_name)) diff --git a/pycross/private/lock_common.bzl b/pycross/private/lock_common.bzl index 34a59fd5..7801db2f 100644 --- a/pycross/private/lock_common.bzl +++ b/pycross/private/lock_common.bzl @@ -412,27 +412,28 @@ def process_workspaces( # Apply Defaulting Rules for ws_name, ws_info in workspaces.items(): - if workspace_repo_count[ws_name] == 0: - discovered = workspace_discovered_members[ws_name] - if len(discovered) == 1: - # Auto-create implicit repo for the only project - project_name = list(discovered.keys())[0] - implicit_tag = struct( - workspace = ws_name, - projects = [project_name], - repo = ws_name, - dependency_groups = ["default"], - flags = [], - settings = {}, - constraint_values = [], - platform = None, - ) - member_tags.append(struct(tag = implicit_tag, module = ws_info.module)) - workspace_repo_count[ws_name] += 1 - elif len(discovered) > 1: - fail("workspace '{}' contains multiple projects but has no repo tags.".format(ws_name)) - else: - fail("workspace '{}' contains no projects.".format(ws_name)) + if workspace_repo_count[ws_name] != 0: + continue + discovered = workspace_discovered_members[ws_name] + if not discovered: + fail("workspace '{}' contains no projects.".format(ws_name)) + if len(discovered) > 1: + fail("workspace '{}' contains multiple projects but has no repo tags.".format(ws_name)) + + # Auto-create implicit repo for the only project + project_name = list(discovered.keys())[0] + implicit_tag = struct( + workspace = ws_name, + projects = [project_name], + repo = ws_name, + dependency_groups = ["default"], + flags = [], + settings = {}, + constraint_values = [], + platform = None, + ) + member_tags.append(struct(tag = implicit_tag, module = ws_info.module)) + workspace_repo_count[ws_name] += 1 for tag_info in member_tags: tag = tag_info.tag @@ -442,17 +443,15 @@ def process_workspaces( projects = getattr(tag, "projects", []) if not projects: - if len(discovered) == 1: - projects = [list(discovered.keys())[0]] - else: + if len(discovered) != 1: fail("projects list is empty but workspace '{}' has {} members".format(ws_name, len(discovered))) + projects = [list(discovered.keys())[0]] repo = tag.repo if not repo: - if workspace_repo_count[ws_name] == 1: - repo = ws_name - else: + if workspace_repo_count[ws_name] != 1: fail("repo name is required for repo tags in workspace '{}' (multiple repo tags exist)".format(ws_name)) + repo = ws_name # Build complete tag new_tag = struct( diff --git a/pycross/private/lock_repo_creation.bzl b/pycross/private/lock_repo_creation.bzl index 54b2d721..a0d1aeb6 100644 --- a/pycross/private/lock_repo_creation.bzl +++ b/pycross/private/lock_repo_creation.bzl @@ -89,6 +89,57 @@ def _validate_override_packages(override_configs, all_resolved_locks, workspace_ ", ".join(sorted(known_names[pkg_name].keys())), )) +def _create_remote_file_repo(key, file, indexes): + remote_file_repo = "pypi_{}".format(sanitize_name(key.replace("/", "_"))) + is_wheel = file["name"].endswith(".whl") + remote_file_label = ( + "@{}//:wheel".format(remote_file_repo) if is_wheel else "@{}//file:{}".format(remote_file_repo, file["name"]) + ) + + urls = file.get("urls", []) + if urls: + if is_wheel: + pycross_wheel_file( + name = remote_file_repo, + urls = urls, + sha256 = file["sha256"], + filename = file["name"], + ) + elif urls[0].startswith("git+"): + pycross_git_file( + name = remote_file_repo, + url = urls[0], + filename = file["name"], + ) + else: + http_file( + name = remote_file_repo, + urls = urls, + sha256 = file["sha256"], + downloaded_file_path = file["name"], + ) + return remote_file_label + + pypi_file_attrs = dict( + name = remote_file_repo, + package_name = file["package_name"], + package_version = file["package_version"], + filename = file["name"], + sha256 = file["sha256"], + ) + + # A per-package index recorded in the lock takes priority over + # the workspace's pypi_indexes. + if file.get("index"): + pypi_file_attrs["indexes"] = [file["index"]] + elif indexes: + pypi_file_attrs["indexes"] = indexes + if is_wheel: + pycross_wheel_file(**pypi_file_attrs) + else: + pypi_file(**pypi_file_attrs) + return remote_file_label + def create_repos( module_ctx, all_locks, @@ -168,60 +219,9 @@ def create_repos( indexes = workspace_pypi_indexes.get(workspace_name, []) if workspace_name else [] for key, file in resolved_lock.get("remote_files", {}).items(): - if key in all_remote_files: - repo_remote_files[key] = all_remote_files[key] - continue - - remote_file_repo = "pypi_{}".format(sanitize_name(key.replace("/", "_"))) - if file["name"].endswith(".whl"): - remote_file_label = "@{}//:wheel".format(remote_file_repo) - else: - remote_file_label = "@{}//file:{}".format(remote_file_repo, file["name"]) - - urls = file.get("urls", []) - if urls: - if file["name"].endswith(".whl"): - pycross_wheel_file( - name = remote_file_repo, - urls = urls, - sha256 = file["sha256"], - filename = file["name"], - ) - elif urls[0].startswith("git+"): - pycross_git_file( - name = remote_file_repo, - url = urls[0], - filename = file["name"], - ) - else: - http_file( - name = remote_file_repo, - urls = urls, - sha256 = file["sha256"], - downloaded_file_path = file["name"], - ) - else: - pypi_file_attrs = dict( - name = remote_file_repo, - package_name = file["package_name"], - package_version = file["package_version"], - filename = file["name"], - sha256 = file["sha256"], - ) - - # A per-package index recorded in the lock takes priority over - # the workspace's pypi_indexes. - if file.get("index"): - pypi_file_attrs["indexes"] = [file["index"]] - elif indexes: - pypi_file_attrs["indexes"] = indexes - if file["name"].endswith(".whl"): - pycross_wheel_file(**pypi_file_attrs) - else: - pypi_file(**pypi_file_attrs) - - repo_remote_files[key] = remote_file_label - all_remote_files[key] = remote_file_label + if key not in all_remote_files: + all_remote_files[key] = _create_remote_file_repo(key, file, indexes) + repo_remote_files[key] = all_remote_files[key] # Pre-calculate known packages in this lock file to filter sdist build_requires known_packages = known_packages_by_repo[repo_name] diff --git a/pycross/private/lock_resolver.bzl b/pycross/private/lock_resolver.bzl index e73ce4f1..833dbc0c 100644 --- a/pycross/private/lock_resolver.bzl +++ b/pycross/private/lock_resolver.bzl @@ -219,14 +219,15 @@ def _create_package_resolver(pkg_key, pkg, ann, default_extra_build_tools, conte marker_dependencies = [] for dep in pkg.get("dependencies", []): dep_name = dep["name"] - if dep_name not in ann_ignore_deps: - dep_key = "{}@{}".format(dep_name, dep["version"]) - dependencies.append(dep) - marker_dependencies.append({ - "key": dep_key, - "marker": dep.get("marker", ""), - }) - all_dependency_keys.append(dep_key) + if dep_name in ann_ignore_deps: + continue + dep_key = "{}@{}".format(dep_name, dep["version"]) + dependencies.append(dep) + marker_dependencies.append({ + "key": dep_key, + "marker": dep.get("marker", ""), + }) + all_dependency_keys.append(dep_key) ann_extra_deps = ann.extra_dependencies if ann else [] for dep_key in ann_extra_deps: @@ -302,6 +303,32 @@ def _create_package_resolver(pkg_key, pkg, ann, default_extra_build_tools, conte key = pkg_key, ) +def _synthesize_extra_package(next_package_key, lock_model_packages, synthesized_packages): + parts = parse_package_key(next_package_key) + if not parts.extra: + fail("Missing package {}".format(next_package_key)) + + base_key = "{}@{}".format(parts.name, parts.version) + base_package = lock_model_packages.get(base_key) or synthesized_packages.get(base_key) + if not base_package: + fail("Missing base package {} for extra {}".format(base_key, next_package_key)) + + base_deps = base_package.get("dependencies", []) + synthesized_deps = list(base_deps) + [{ + "name": parts.name, + "version": parts.version, + "marker": "", + }] + raw_pkg = { + "name": parts.name, + "version": parts.version, + "python_versions": base_package.get("python_versions", ""), + "dependencies": synthesized_deps, + "files": [], + } + synthesized_packages[next_package_key] = raw_pkg + return raw_pkg + def _resolve_packages( lock_model_packages, pins, @@ -337,29 +364,11 @@ def _resolve_packages( raw_pkg = lock_model_packages.get(next_package_key) or synthesized_packages.get(next_package_key) if not raw_pkg: - parts = parse_package_key(next_package_key) - if parts.extra: - base_key = "{}@{}".format(parts.name, parts.version) - base_package = lock_model_packages.get(base_key) or synthesized_packages.get(base_key) - if not base_package: - fail("Missing base package {} for extra {}".format(base_key, next_package_key)) - - base_deps = base_package.get("dependencies", []) - synthesized_deps = list(base_deps) + [{ - "name": parts.name, - "version": parts.version, - "marker": "", - }] - raw_pkg = { - "name": parts.name, - "version": parts.version, - "python_versions": base_package.get("python_versions", ""), - "dependencies": synthesized_deps, - "files": [], - } - synthesized_packages[next_package_key] = raw_pkg - else: - fail("Missing package {}".format(next_package_key)) + raw_pkg = _synthesize_extra_package( + next_package_key, + lock_model_packages, + synthesized_packages, + ) ann = annotations.pop(next_package_key, None) entry = _create_package_resolver( @@ -445,24 +454,25 @@ def _compute_cycle_groups(packages): work_stack.append([w, graph.get(w, []), 0]) elif w in on_stack: lowlink[v] = min(lowlink[v], indices[w]) - else: - if lowlink[v] == indices[v]: - scc = [] - - # We also need to simulate this while True loop - # It runs at most len(stack) times. - for _ in range(len(stack) + 1): - w = stack.pop() - on_stack.pop(w) - scc.append(w) - if w == v: - break - sccs.append(scc) - - work_stack.pop() - if len(work_stack) > 0: - parent = work_stack[-1][0] - lowlink[parent] = min(lowlink[parent], lowlink[v]) + continue + + if lowlink[v] == indices[v]: + scc = [] + + # We also need to simulate this while True loop + # It runs at most len(stack) times. + for _ in range(len(stack) + 1): + w = stack.pop() + on_stack.pop(w) + scc.append(w) + if w == v: + break + sccs.append(scc) + + work_stack.pop() + if len(work_stack) > 0: + parent = work_stack[-1][0] + lowlink[parent] = min(lowlink[parent], lowlink[v]) if len(work_stack) > 0: fail("DFS traversal for SCC exceeded max iterations. Remaining stack: {}".format(work_stack)) diff --git a/pycross/private/modules_mapping.bzl b/pycross/private/modules_mapping.bzl index dc56e3aa..4274050a 100644 --- a/pycross/private/modules_mapping.bzl +++ b/pycross/private/modules_mapping.bzl @@ -14,29 +14,25 @@ def _is_valid_identifier(s): Returns: True if the string is a valid identifier, False otherwise. """ - if not s: + if not s or not (s[0].isalpha() or s[0] == "_"): return False - is_first = True - for c in s.elems(): - if is_first: - if not (c.isalpha() or c == "_"): - return False - is_first = False - elif not (c.isalnum() or c == "_"): + for c in s[1:].elems(): + if not (c.isalnum() or c == "_"): return False return True def _strip_extensions(module_name): for ext in [".pth", ".so", ".py", ".pyd"]: - if module_name.endswith(ext): - module_name = module_name[:-len(ext)] - if ".cpython-" in module_name: - module_name = module_name.split(".cpython-")[0] - elif ".abi3-" in module_name: - module_name = module_name.split(".abi3-")[0] - elif module_name.endswith(".abi3"): - module_name = module_name[:-5] - break + if not module_name.endswith(ext): + continue + module_name = module_name[:-len(ext)] + if ".cpython-" in module_name: + module_name = module_name.split(".cpython-")[0] + elif ".abi3-" in module_name: + module_name = module_name.split(".abi3-")[0] + elif module_name.endswith(".abi3"): + module_name = module_name[:-5] + break return module_name def _is_valid_module_name(module_name): @@ -56,14 +52,15 @@ def _pycross_modules_mapping_impl(ctx): mapping = {} for dep in ctx.attr.deps: - if PycrossPackageInfo in dep: - pkg_info = dep[PycrossPackageInfo] - for tlp in pkg_info.site_paths: - module_name = _get_module_name(tlp) - if not module_name: - continue + if PycrossPackageInfo not in dep: + continue + pkg_info = dep[PycrossPackageInfo] + for tlp in pkg_info.site_paths: + module_name = _get_module_name(tlp) + if not module_name: + continue - mapping[module_name] = pkg_info.package_name.replace("-", "_") + mapping[module_name] = pkg_info.package_name.replace("-", "_") out = ctx.actions.declare_file(ctx.attr.name + ".json") ctx.actions.write(out, json.encode(mapping)) diff --git a/pycross/private/pep508_marker_values.bzl b/pycross/private/pep508_marker_values.bzl index 759a7e01..048bbe01 100644 --- a/pycross/private/pep508_marker_values.bzl +++ b/pycross/private/pep508_marker_values.bzl @@ -184,6 +184,14 @@ def marker_value_attrs(): "_platform_python_implementation_target": attr.label(default = "@rules_pycross//pycross/settings:pep508_platform_python_implementation"), } +def _toolchain_py3_runtime(ctx): + if PYTHON_TOOLCHAIN_TYPE not in ctx.toolchains: + return None + tc = ctx.toolchains[PYTHON_TOOLCHAIN_TYPE] + if not hasattr(tc, "py3_runtime"): + return None + return tc.py3_runtime + def collect_markers(ctx): """Collects PEP 508 marker values from rule context attrs and toolchain. @@ -216,26 +224,20 @@ def collect_markers(ctx): platform_python_implementation = ctx.attr.platform_python_implementation # Try PyRuntimeInfo from the toolchain first (most accurate). - if (not python_version or not python_full_version or not implementation_name): - runtime = None - if PYTHON_TOOLCHAIN_TYPE in ctx.toolchains: - tc = ctx.toolchains[PYTHON_TOOLCHAIN_TYPE] - if hasattr(tc, "py3_runtime"): - runtime = tc.py3_runtime - - if runtime: - if not implementation_name and hasattr(runtime, "implementation_name") and runtime.implementation_name: - implementation_name = runtime.implementation_name - - if hasattr(runtime, "interpreter_version_info") and runtime.interpreter_version_info: - vi = runtime.interpreter_version_info - if not python_version: - python_version = "{}.{}".format(vi.major, vi.minor) - full_ver = _format_full_version(vi) - if not python_full_version: - python_full_version = full_ver - if not implementation_version: - implementation_version = full_ver + runtime = _toolchain_py3_runtime(ctx) if (not python_version or not python_full_version or not implementation_name) else None + if runtime: + if not implementation_name and hasattr(runtime, "implementation_name") and runtime.implementation_name: + implementation_name = runtime.implementation_name + + vi = runtime.interpreter_version_info if hasattr(runtime, "interpreter_version_info") else None + if vi: + if not python_version: + python_version = "{}.{}".format(vi.major, vi.minor) + full_ver = _format_full_version(vi) + if not python_full_version: + python_full_version = full_ver + if not implementation_version: + implementation_version = full_ver # Fall back to target flags. if not python_version: diff --git a/pycross/private/poetry_lock_model.bzl b/pycross/private/poetry_lock_model.bzl index dc605382..a78dd9de 100644 --- a/pycross/private/poetry_lock_model.bzl +++ b/pycross/private/poetry_lock_model.bzl @@ -218,15 +218,15 @@ def _get_files_for_package(files, package_name, package_version): if filename.endswith(".whl"): # Parse wheel filename to check name and version parsed = pypackaging.utils.parse_wheel_filename(filename) - if parsed.name == package_name and pypackaging.version.to_string(parsed.version) == package_version: - result.append(f) elif filename.endswith(".tar.gz") or filename.endswith(".zip"): parsed = pypackaging.utils.parse_sdist_filename(filename) - if parsed.name == package_name and pypackaging.version.to_string(parsed.version) == package_version: - result.append(f) else: # Unknown format, include it result.append(f) + continue + + if parsed.name == package_name and pypackaging.version.to_string(parsed.version) == package_version: + result.append(f) return result @@ -417,39 +417,41 @@ def translate_poetry(project_dict, lock_dict, lock_model): for group_name in project_optional_deps.keys(): key = "optional:{}".format(group_name) - if key in requested_groups_dict: - is_testonly = requested_groups_dict[key] - for dep_str in project_optional_deps[group_name]: + if key not in requested_groups_dict: + continue + is_testonly = requested_groups_dict[key] + for dep_str in project_optional_deps[group_name]: + req = parse_pep508_requirement(dep_str) + if req.name == "python": + continue + track_pin(req.name, req.specifier, is_testonly, req.marker, req.extras) + + for group_name in available_dev_groups: + key = "group:{}".format(group_name) + if key not in requested_groups_dict: + continue + is_testonly = requested_groups_dict[key] + + # Poetry merges PEP 735 and legacy groups (union, not fallback). + if group_name in pep735_groups: + for dep_str in pep735_groups[group_name]: req = parse_pep508_requirement(dep_str) if req.name == "python": continue - track_pin(req.name, req.specifier, is_testonly, req.marker, req.extras) + track_pin(canonicalize_name(req.name), req.specifier, is_testonly, req.marker, req.extras) - for group_name in available_dev_groups: - key = "group:{}".format(group_name) - if key in requested_groups_dict: - is_testonly = requested_groups_dict[key] - - # Poetry merges PEP 735 and legacy groups (union, not fallback). - if group_name in pep735_groups: - for dep_str in pep735_groups[group_name]: - req = parse_pep508_requirement(dep_str) - if req.name == "python": - continue - track_pin(canonicalize_name(req.name), req.specifier, is_testonly, req.marker, req.extras) - - if group_name in poetry_groups: - g = poetry_groups[group_name] - for pin, pin_info in g.get("dependencies", {}).items(): - pin = canonicalize_name(pin) - if pin == "python": - continue - _parse_poetry_pin( - pin, - pin_info, - pinned_package_specs, - track_pin = lambda p, spec, marker, extras: track_pin(p, spec, is_testonly, marker, extras), - ) + if group_name in poetry_groups: + g = poetry_groups[group_name] + for pin, pin_info in g.get("dependencies", {}).items(): + pin = canonicalize_name(pin) + if pin == "python": + continue + _parse_poetry_pin( + pin, + pin_info, + pinned_package_specs, + track_pin = lambda p, spec, marker, extras: track_pin(p, spec, is_testonly, marker, extras), + ) testonly_pin_names = [name for name in testonly_reqs if name not in non_testonly_reqs] diff --git a/pycross/private/pylock_lock_model.bzl b/pycross/private/pylock_lock_model.bzl index 092f6372..8d3ac344 100644 --- a/pycross/private/pylock_lock_model.bzl +++ b/pycross/private/pylock_lock_model.bzl @@ -104,6 +104,139 @@ def _link_through_hub(lock_packages, versions_all): if len(hubs) == 1: _set_dependency(pkg, lock_packages[hub], "") +def _resolve_pylock_project_pins(project_dict, lock_model, lock_packages, deps_by_name, pkg_markers, versions_all): + """Filter lock_packages and compute pins, testonly_pins, and root_dependency_markers.""" + if not project_dict: + # Default: include all (use first key per name; fork detection below may override) + pins = {} + for pkg_key, pkg in lock_packages.items(): + pins.setdefault(pkg["name"], pkg_key) + return lock_packages, pins, [], {} + + dependency_groups = getattr(lock_model, "dependency_groups", ["default"]) + testonly_groups = getattr(lock_model, "testonly_groups", []) + non_testonly_groups = getattr(lock_model, "non_testonly_groups", []) + wildcard_testonly = getattr(lock_model, "wildcard_testonly", False) + + project_section = project_dict.get("project", {}) + optional_deps = project_section.get("optional-dependencies", {}) + dev_deps = project_dict.get("dependency-groups", {}) + project_name = project_section.get("name") + + requested_groups_dict = compute_requested_dependency_groups( + dependency_groups = dependency_groups, + testonly_groups = testonly_groups, + non_testonly_groups = non_testonly_groups, + wildcard_testonly = wildcard_testonly, + available_groups = ( + ["default"] + + ["optional:" + g for g in optional_deps.keys()] + + ["group:" + g for g in dev_deps.keys()] + ), + project_name = project_name, + ) + + root_req_names = [] + testonly_root_req_names = [] + root_dependency_markers = {} + + def handle_dep_str(dep_str, is_testonly): + req = parse_pep508_requirement(dep_str) + n = req.name + if is_testonly: + testonly_root_req_names.append(n) + else: + root_req_names.append(n) + for extra in req.extras: + pin_name = "{}[{}]".format(n, canonicalize_name(extra)) + record_root_marker(root_dependency_markers, pin_name, req.marker) + record_root_marker(root_dependency_markers, n, req.marker) + + if "default" in requested_groups_dict: + is_testonly = requested_groups_dict["default"] + for dep_str in project_section.get("dependencies", []): + handle_dep_str(dep_str, is_testonly) + + for kind, groups_dict in [("optional", optional_deps), ("group", dev_deps)]: + for target_name, entries in groups_dict.items(): + key = "{}:{}".format(kind, target_name) + if key not in requested_groups_dict: + continue + is_testonly = requested_groups_dict[key] + for entry in entries: + if type(entry) == "string": + handle_dep_str(entry, is_testonly) + elif type(entry) == "dict" and "include-group" in entry: + for inc_dep in dev_deps.get(entry["include-group"], []): + if type(inc_dep) == "string": + handle_dep_str(inc_dep, is_testonly) + + # Deduplicate + root_package_names = {n: True for n in root_req_names} + testonly_package_names = {n: True for n in testonly_root_req_names if n not in root_package_names} + + # BFS from all root names to find reachable packages + visited_names = {} + all_roots = dict(root_package_names) + all_roots.update(testonly_package_names) + queue = sorted(all_roots.keys()) + + # Starlark has no while loop, simulate with for+range. + # Upper bound: each edge can add one item to the queue, plus the initial queue size. + total_edges = 0 + for pkg_key in lock_packages: + total_edges += len(lock_packages[pkg_key].get("dependencies", [])) + max_iter = total_edges + len(queue) + 1 + for _ in range(max_iter): + if not queue: + break + curr = queue[0] + queue = queue[1:] + if curr in visited_names: + continue + visited_names[curr] = True + + for pkg_key in deps_by_name.get(curr, []): + if pkg_key not in lock_packages: + continue + for dep in lock_packages[pkg_key].get("dependencies", []): + queue.append(parse_package_key(dep["name"]).name) + + if queue: + fail("BFS traversal exceeded max iterations; this is a bug in pycross") + + # PDM's export drops extras edges (requests[socks] -> pysocks), leaving + # locked packages unreachable from the roots. If any such orphan's PDM + # selection marker names a requested group, model the lock as graphless + # instead. Other unreachable packages are pruned as before. + orphans = { + k: True + for k, pkg in lock_packages.items() + if pkg["name"] not in visited_names and _selects_requested_group(pkg_markers[k], requested_groups_dict) + } + if orphans: + lock_packages = { + k: pkg + for k, pkg in lock_packages.items() + if k in orphans or pkg["name"] in visited_names + } + _link_through_hub(lock_packages, versions_all) + visited_names = {pkg["name"]: True for pkg in lock_packages.values()} + + lock_packages = { + pkg_key: pkg + for pkg_key, pkg in lock_packages.items() + if pkg["name"] in visited_names + } + + pins = {} + for root_name in sorted(all_roots.keys()): + keys = deps_by_name.get(root_name, []) + if keys: + pins[root_name] = keys[0] + + return lock_packages, pins, sorted(testonly_package_names.keys()), root_dependency_markers + def translate_pylock(lock_dict, project_dict, lock_model): """Translates pylock data to raw_lock_data dict. @@ -294,147 +427,14 @@ def translate_pylock(lock_dict, project_dict, lock_model): pkg_name = pkg["name"] deps_by_name.setdefault(pkg_name, []).append(pkg_key) - pins = {} - - dependency_groups = getattr(lock_model, "dependency_groups", ["default"]) - root_dependency_markers = {} - - if project_dict: - root_req_names = [] - testonly_root_req_names = [] - testonly_groups = getattr(lock_model, "testonly_groups", []) - non_testonly_groups = getattr(lock_model, "non_testonly_groups", []) - wildcard_testonly = getattr(lock_model, "wildcard_testonly", False) - - project_section = project_dict.get("project", {}) - optional_deps = project_section.get("optional-dependencies", {}) - dev_deps = project_dict.get("dependency-groups", {}) - - project_name = project_dict.get("project", {}).get("name") - - requested_groups_dict = compute_requested_dependency_groups( - dependency_groups = dependency_groups, - testonly_groups = testonly_groups, - non_testonly_groups = non_testonly_groups, - wildcard_testonly = wildcard_testonly, - available_groups = ( - ["default"] + - ["optional:" + g for g in optional_deps.keys()] + - ["group:" + g for g in dev_deps.keys()] - ), - project_name = project_name, - ) - - def handle_dep_str(dep_str, is_testonly): - req = parse_pep508_requirement(dep_str) - n = req.name - if is_testonly: - testonly_root_req_names.append(n) - else: - root_req_names.append(n) - if req.extras: - for extra in req.extras: - pin_name = "{}[{}]".format(n, canonicalize_name(extra)) - record_root_marker(root_dependency_markers, pin_name, req.marker) - record_root_marker(root_dependency_markers, n, req.marker) - - if "default" in requested_groups_dict: - is_testonly = requested_groups_dict["default"] - for dep_str in project_section.get("dependencies", []): - handle_dep_str(dep_str, is_testonly) - - for kind, groups_dict in [("optional", optional_deps), ("group", dev_deps)]: - for target_name in groups_dict.keys(): - key = "{}:{}".format(kind, target_name) - if key not in requested_groups_dict: - continue - is_testonly = requested_groups_dict[key] - entries = groups_dict[target_name] - for entry in entries: - if type(entry) == "string": - handle_dep_str(entry, is_testonly) - elif type(entry) == "dict" and "include-group" in entry: - inc_group = entry["include-group"] - if inc_group in dev_deps: - for inc_dep in dev_deps[inc_group]: - if type(inc_dep) == "string": - handle_dep_str(inc_dep, is_testonly) - - # Deduplicate - root_package_names = {n: True for n in root_req_names} - testonly_package_names = {n: True for n in testonly_root_req_names if n not in root_package_names} - - # BFS from all root names to find reachable packages - visited_names = {} - all_roots = dict(root_package_names) - all_roots.update(testonly_package_names) - queue = sorted(all_roots.keys()) - - # Starlark has no while loop, simulate with for+range. - # Upper bound: each edge can add one item to the queue, plus the initial queue size. - total_edges = 0 - for pkg_key in lock_packages: - total_edges += len(lock_packages[pkg_key].get("dependencies", [])) - max_iter = total_edges + len(queue) + 1 - for _ in range(max_iter): - if not queue: - break - curr = queue[0] - queue = queue[1:] - if curr in visited_names: - continue - visited_names[curr] = True - - for pkg_key in deps_by_name.get(curr, []): - if pkg_key in lock_packages: - for dep in lock_packages[pkg_key].get("dependencies", []): - dep_base = parse_package_key(dep["name"]).name - queue.append(dep_base) - - if queue: - fail("BFS traversal exceeded max iterations; this is a bug in pycross") - - # PDM's export drops extras edges (requests[socks] -> pysocks), leaving - # locked packages unreachable from the roots. If any such orphan's PDM - # selection marker names a requested group, model the lock as graphless - # instead. Other unreachable packages are pruned as before. - orphans = { - k: True - for k, pkg in lock_packages.items() - if pkg["name"] not in visited_names and _selects_requested_group(pkg_markers[k], requested_groups_dict) - } - if orphans: - lock_packages = { - k: pkg - for k, pkg in lock_packages.items() - if k in orphans or pkg["name"] in visited_names - } - _link_through_hub(lock_packages, versions_all) - visited_names = {pkg["name"]: True for pkg in lock_packages.values()} - - # Filter - filtered_packages = {} - for pkg_key, pkg in lock_packages.items(): - if pkg["name"] in visited_names: - filtered_packages[pkg_key] = pkg - lock_packages = filtered_packages - - # Pins from roots - for root_name in sorted(all_roots.keys()): - keys = deps_by_name.get(root_name, []) - if keys: - pins[root_name] = keys[0] - - testonly_pins = sorted(testonly_package_names.keys()) - else: - # Default: include all (use first key per name; fork detection below may override) - seen_names = {} - for pkg_key, pkg in lock_packages.items(): - pname = pkg["name"] - if pname not in seen_names: - pins[pname] = pkg_key - seen_names[pname] = True - testonly_pins = [] + lock_packages, pins, testonly_pins, root_dependency_markers = _resolve_pylock_project_pins( + project_dict, + lock_model, + lock_packages, + deps_by_name, + pkg_markers, + versions_all, + ) # Detect resolution-marker forks: same package name with multiple versions. resolution_marker_exprs = {} @@ -451,11 +451,12 @@ def translate_pylock(lock_dict, project_dict, lock_model): # If forks were detected, update pins to use conditional pins. for fname, version_constraints in fork_constraints.items(): - if fname in pins: - conditional_pins = {} - for fversion, cname in version_constraints.items(): - conditional_pins[cname] = "{}@{}".format(fname, fversion) - pins[fname] = conditional_pins + if fname not in pins: + continue + pins[fname] = { + cname: "{}@{}".format(fname, fversion) + for fversion, cname in version_constraints.items() + } result = { "packages": lock_packages, diff --git a/pycross/private/pypi_file.bzl b/pycross/private/pypi_file.bzl index c7625049..2bb622c8 100644 --- a/pycross/private/pypi_file.bzl +++ b/pycross/private/pypi_file.bzl @@ -43,6 +43,21 @@ def _sha256_from_fragment(url): return part[len("sha256="):].lower() return None +def _extract_href(tag): + tag_lower = tag.lower() + idx = tag_lower.find("href") + for _attempt in range(len(tag)): + if idx < 0: + return None + rest = tag[idx + 4:].lstrip() + if (idx == 0 or tag[idx - 1] in " \t\r\n") and rest.startswith("="): + rest = rest[1:].lstrip() + if rest[:1] in ("\"", "'"): + return rest[1:].split(rest[0], 1)[0] + return rest.replace("\t", " ").replace("\r", " ").replace("\n", " ").split(" ", 1)[0] + idx = tag_lower.find("href", idx + 4) + return None + def _parse_simple_html(content): """Parses a PEP 503 project page into a list of file dicts.""" files = [] @@ -65,21 +80,7 @@ def _parse_simple_html(content): text = content[end + 1:close] pos = close + 4 - href = None - tag_lower = tag.lower() - idx = tag_lower.find("href") - for _attempt in range(len(tag)): - if idx < 0: - break - rest = tag[idx + 4:].lstrip() - if (idx == 0 or tag[idx - 1] in " \t\r\n") and rest.startswith("="): - rest = rest[1:].lstrip() - if rest[:1] in ("\"", "'"): - href = rest[1:].split(rest[0], 1)[0] - else: - href = rest.replace("\t", " ").replace("\r", " ").replace("\n", " ").split(" ", 1)[0] - break - idx = tag_lower.find("href", idx + 4) + href = _extract_href(tag) if href == None: continue href = _html_unescape(href.strip()) diff --git a/pycross/private/resolved_lock_renderer.bzl b/pycross/private/resolved_lock_renderer.bzl index 68414f21..6c4e103a 100644 --- a/pycross/private/resolved_lock_renderer.bzl +++ b/pycross/private/resolved_lock_renderer.bzl @@ -41,11 +41,9 @@ def _render_extras_aggregates(lines, packages): base_packages_with_extras = {} for pkg_key in packages.keys(): parts = parse_package_key(pkg_key) - if parts.extra: - base_key = (parts.name, parts.version) - if base_key not in base_packages_with_extras: - base_packages_with_extras[base_key] = [] - base_packages_with_extras[base_key].append(pkg_key) + if not parts.extra: + continue + base_packages_with_extras.setdefault((parts.name, parts.version), []).append(pkg_key) for (base_name, version), extra_keys in sorted(base_packages_with_extras.items()): base_pkg_key = "{}@{}".format(base_name, version) @@ -74,11 +72,13 @@ def _collect_unique_markers(packages): extra = parse_package_key(pkg_key).extra for md in pkg.get("marker_dependencies", []): marker = md.get("marker") - if marker: - # Optimization: if "extra" is not in the marker string, it doesn't reference the extra variable. - # We can share a single evaluator by setting extra to "". - effective_extra = extra if "extra" in marker else "" - markers[(marker, effective_extra)] = True + if not marker: + continue + + # Optimization: if "extra" is not in the marker string, it doesn't reference the extra variable. + # We can share a single evaluator by setting extra to "". + effective_extra = extra if "extra" in marker else "" + markers[(marker, effective_extra)] = True # Availability markers come from root project dependency edges, which # never reference `extra`, so they share the extra-free evaluators. @@ -190,11 +190,12 @@ def _build_cycle_edges_dict(scc, packages): edge_list = [] for md in pkg.get("marker_dependencies", []): dep_key = md["key"] - if dep_key in scc_set: - entry = {"dep": dep_key} - if md.get("marker"): - entry["marker"] = md["marker"] - edge_list.append(entry) + if dep_key not in scc_set: + continue + entry = {"dep": dep_key} + if md.get("marker"): + entry["marker"] = md["marker"] + edge_list.append(entry) edges[pkg_key] = edge_list return edges @@ -341,27 +342,28 @@ def _render_marker_wheel_chooser(lines, pkg_key, pkg, repo_map, sdist_target = N # platforms. When the package also carries availability markers, the group # is named _wheel_available_ so _render_marker_availability can compose it # with the marker-based group into the final _available_ group. - if not sdist_target and resolved_candidates: - wheel_gated = bool(pkg.get("availability_markers")) - group_name = "_wheel_available_{}".format(pkg_key) if wheel_gated else "_available_{}".format(pkg_key) - cs_names = [] - for candidate, _target in resolved_candidates: - filename = candidate["filename"] - cs_names.append(":_wheel_cs_{}_{}".format(pkg_key, _sanitize_name(filename))) - lines.extend([ - _ind("selects.config_setting_group("), - _ind('name = "{}",'.format(group_name), 2), - _ind("match_any = [", 2), - ]) - for cs in cs_names: - lines.append(_ind('"{}",'.format(cs), 3)) - lines.extend([ - _ind("],", 2), - _ind(")"), - "", - ]) - return wheel_gated - return False + if sdist_target or not resolved_candidates: + return False + + wheel_gated = bool(pkg.get("availability_markers")) + group_name = "_wheel_available_{}".format(pkg_key) if wheel_gated else "_available_{}".format(pkg_key) + cs_names = [] + for candidate, _target in resolved_candidates: + filename = candidate["filename"] + cs_names.append(":_wheel_cs_{}_{}".format(pkg_key, _sanitize_name(filename))) + lines.extend([ + _ind("selects.config_setting_group("), + _ind('name = "{}",'.format(group_name), 2), + _ind("match_any = [", 2), + ]) + for cs in cs_names: + lines.append(_ind('"{}",'.format(cs), 3)) + lines.extend([ + _ind("],", 2), + _ind(")"), + "", + ]) + return wheel_gated def _render_marker_package_deps(lines, pkg_key, pkg_key_san, pkg, packages): """Render deps using marker-based select() instead of environment-based.""" diff --git a/pycross/private/sdist_repo.bzl b/pycross/private/sdist_repo.bzl index c4892f74..2d096a72 100644 --- a/pycross/private/sdist_repo.bzl +++ b/pycross/private/sdist_repo.bzl @@ -129,11 +129,12 @@ pycross_wheel_metadata( rctx.file("REPO.bazel", "") def _validate_build_backend(attr): - if attr.build_backend: - known_backends = {v: True for v in attr.backend_to_rule.values()} - if attr.build_backend not in known_backends and attr.build_backend != attr.default_backend: - fail("Unknown build backend: " + attr.build_backend + - ". Registered backends: " + ", ".join(sorted(known_backends.keys()))) + if not attr.build_backend: + return + known_backends = {v: True for v in attr.backend_to_rule.values()} + if attr.build_backend not in known_backends and attr.build_backend != attr.default_backend: + fail("Unknown build backend: " + attr.build_backend + + ". Registered backends: " + ", ".join(sorted(known_backends.keys()))) # Tool package names that every build rule accepts in tool_deps, in addition to the # backend's registered tool_packages. @@ -312,33 +313,34 @@ def _validate_requirements(requires, pin_versions, pkg_name): if req_name == "oldest-supported-numpy": req_name = "numpy" - if req_name in normalized_pins: - version_or_dict = normalized_pins[req_name] - if type(version_or_dict) == "string": - if not pypackaging.specifiers.set_contains(req.specifier, version_or_dict): - warnings.append( - "WARNING: The build tools repo pins '{}=={}', but '{}' requires '{}' in pyproject.toml.".format( - req_name, - version_or_dict, - pkg_name, - req_str, - ), - ) - elif type(version_or_dict) == "dict": - satisfying = [ - v - for v in version_or_dict.values() - if pypackaging.specifiers.set_contains(req.specifier, v) - ] - if not satisfying: - versions_str = ", ".join(["{}={}".format(k, v) for k, v in sorted(version_or_dict.items())]) - warnings.append( - "WARNING: No variant of '{}' satisfies '{}' (available: {}).".format( - req_name, - req_str, - versions_str, - ), - ) + if req_name not in normalized_pins: + continue + version_or_dict = normalized_pins[req_name] + if type(version_or_dict) == "string": + if not pypackaging.specifiers.set_contains(req.specifier, version_or_dict): + warnings.append( + "WARNING: The build tools repo pins '{}=={}', but '{}' requires '{}' in pyproject.toml.".format( + req_name, + version_or_dict, + pkg_name, + req_str, + ), + ) + elif type(version_or_dict) == "dict": + satisfying = [ + v + for v in version_or_dict.values() + if pypackaging.specifiers.set_contains(req.specifier, v) + ] + if not satisfying: + versions_str = ", ".join(["{}={}".format(k, v) for k, v in sorted(version_or_dict.items())]) + warnings.append( + "WARNING: No variant of '{}' satisfies '{}' (available: {}).".format( + req_name, + req_str, + versions_str, + ), + ) return warnings diff --git a/pycross/private/thin_package_repo.bzl b/pycross/private/thin_package_repo.bzl index 2fe97b17..ec6a9dd9 100644 --- a/pycross/private/thin_package_repo.bzl +++ b/pycross/private/thin_package_repo.bzl @@ -202,26 +202,26 @@ def _emit_maybe_alias(lines, maybe_name, pkg_target, target_dict, available_keys emitted_helpers = {} branch_targets = {} for constraint, pkg_key in target_dict.items(): - if pkg_key in available_set: - helper_name = "{}_{}".format( - helper_prefix, - pkg_key.replace("@", "_").replace("[", "_").replace("]", "_"), - ) - if pkg_key not in emitted_helpers: - emitted_helpers[pkg_key] = helper_name - lines.extend([ - "alias(", - ' name = "{}",'.format(helper_name), - " actual = select({", - ' "@{}//_lock:_available_{}": "{}",'.format(workspace_repo, pkg_key, pkg_target), - ' "//conditions:default": "//:_empty_library",', - " }),", - ")", - "", - ]) - branch_targets[constraint] = ":{}".format(emitted_helpers[pkg_key]) - else: + if pkg_key not in available_set: branch_targets[constraint] = pkg_target + continue + helper_name = "{}_{}".format( + helper_prefix, + pkg_key.replace("@", "_").replace("[", "_").replace("]", "_"), + ) + if pkg_key not in emitted_helpers: + emitted_helpers[pkg_key] = helper_name + lines.extend([ + "alias(", + ' name = "{}",'.format(helper_name), + " actual = select({", + ' "@{}//_lock:_available_{}": "{}",'.format(workspace_repo, pkg_key, pkg_target), + ' "//conditions:default": "//:_empty_library",', + " }),", + ")", + "", + ]) + branch_targets[constraint] = ":{}".format(emitted_helpers[pkg_key]) lines.extend([ "alias(", @@ -457,13 +457,11 @@ def _thin_package_repo_impl(rctx): grouped_pins = {} for pin_name, pin_target in pins.items(): parts = parse_package_key(pin_name) + if parts.name not in grouped_pins: + grouped_pins[parts.name] = {"base_target": None, "extras": {}} if parts.extra: - if parts.name not in grouped_pins: - grouped_pins[parts.name] = {"base_target": None, "extras": {}} grouped_pins[parts.name]["extras"][parts.extra] = pin_target else: - if parts.name not in grouped_pins: - grouped_pins[parts.name] = {"base_target": None, "extras": {}} grouped_pins[parts.name]["base_target"] = pin_target root_build_lines = [ @@ -732,9 +730,9 @@ pycross_transitioning_file_proxy = rule( base_packages_with_extras = {} for pkg_key in packages.keys(): parts = parse_package_key(pkg_key) - if parts.extra: - base_pkg_key = "{}@{}".format(parts.name, parts.version) - base_packages_with_extras[base_pkg_key] = True + if not parts.extra: + continue + base_packages_with_extras["{}@{}".format(parts.name, parts.version)] = True # Pin directories: proxies pointing to @workspace//_lock targets for base_pin_name, group in sorted(grouped_pins.items()): diff --git a/pycross/private/tools/inspect_package.py b/pycross/private/tools/inspect_package.py index aabf80f0..3b7b4937 100644 --- a/pycross/private/tools/inspect_package.py +++ b/pycross/private/tools/inspect_package.py @@ -181,7 +181,7 @@ def _matches(name: str) -> bool: return len(p.parts) >= 2 and p.is_relative_to(PurePosixPath(p.parts[0]) / target_suffix) return p == target_suffix or (len(p.parts) >= 2 and PurePosixPath(*p.parts[1:]) == target_suffix) - if archive_path.name.endswith(".zip") or archive_path.name.endswith(".whl"): + if archive_path.name.endswith((".zip", ".whl")): with zipfile.ZipFile(archive_path) as z: for name in z.namelist(): if _matches(name): @@ -189,10 +189,11 @@ def _matches(name: str) -> bool: elif archive_path.name.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tar")): with tarfile.open(archive_path) as t: for member in t.getmembers(): - if member.isfile() and _matches(member.name): - f = t.extractfile(member) - if f: - return f.read().decode("utf-8") + if not (member.isfile() and _matches(member.name)): + continue + f = t.extractfile(member) + if f: + return f.read().decode("utf-8") return "" @@ -314,19 +315,22 @@ def _read_top_level_txt(sdist_path: Path) -> list[str] | None: if sdist_path.name.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tar")): with tarfile.open(sdist_path) as t: for member in t.getmembers(): - if member.isfile() and member.name.endswith(".egg-info/top_level.txt"): - f = t.extractfile(member) - if f: - content = f.read().decode("utf-8") - names = [line.strip() for line in content.splitlines() if line.strip()] - break + if not (member.isfile() and member.name.endswith(".egg-info/top_level.txt")): + continue + f = t.extractfile(member) + if not f: + continue + content = f.read().decode("utf-8") + names = [line.strip() for line in content.splitlines() if line.strip()] + break elif sdist_path.name.endswith(".zip"): with zipfile.ZipFile(sdist_path) as z: for name in z.namelist(): - if name.endswith(".egg-info/top_level.txt"): - content = z.read(name).decode("utf-8") - names = [line.strip() for line in content.splitlines() if line.strip()] - break + if not name.endswith(".egg-info/top_level.txt"): + continue + content = z.read(name).decode("utf-8") + names = [line.strip() for line in content.splitlines() if line.strip()] + break return names if names else None @@ -530,12 +534,13 @@ def _inspect_patched_sdist( top_level_names: list[str] | None = None if not source_dir: for p in sorted(sdist_root_dir.rglob("*.egg-info/top_level.txt")): - if p.is_file(): - tl_content = p.read_text(encoding="utf-8") - names = [line.strip() for line in tl_content.splitlines() if line.strip()] - if names: - top_level_names = names - break + if not p.is_file(): + continue + tl_content = p.read_text(encoding="utf-8") + names = [line.strip() for line in tl_content.splitlines() if line.strip()] + if names: + top_level_names = names + break def read_extracted(name: str) -> Optional[str]: return (tmp_path / name).read_text(encoding="utf-8", errors="replace") @@ -587,14 +592,15 @@ def _find_wheel_paths(wheel_path: Path) -> tuple[list[str], list[str], list[str] root_files.add(parts[2]) continue - if parts[0] and not parts[0].endswith(".data"): - if not name.endswith("/"): - all_files.add(name) - members[name] = name - if len(parts) >= 2: - top_level_dirs.add(parts[0]) - elif len(parts) == 1 and parts[0] and not name.endswith("/"): - root_files.add(parts[0]) + if not parts[0] or parts[0].endswith(".data"): + continue + if not name.endswith("/"): + all_files.add(name) + members[name] = name + if len(parts) >= 2: + top_level_dirs.add(parts[0]) + elif len(parts) == 1 and parts[0] and not name.endswith("/"): + root_files.add(parts[0]) def read_file(path: str) -> Optional[str]: return z.read(members[path]).decode("utf-8", "replace") if path in members else None diff --git a/pycross/private/translator_common.bzl b/pycross/private/translator_common.bzl index 19978462..cbd86537 100644 --- a/pycross/private/translator_common.bzl +++ b/pycross/private/translator_common.bzl @@ -89,6 +89,29 @@ def sha256_from_string(s): result = result + s return result[:64] +def _read_project_dict(rctx, extra_project_files, lock_file, lock_model): + """Select and decode the relevant pyproject.toml as a dict.""" + sibling_label = lock_file.relative(":pyproject.toml") + if not extra_project_files: + project_path = rctx.path(sibling_label) + return decode(rctx.read(project_path)) if project_path.exists else {} + + projects = getattr(lock_model, "projects", []) + if len(projects) == 1 and projects[0] != "*": + target_name = canonicalize_name(projects[0]) + for f in extra_project_files: + path = rctx.path(f) + if not path.exists: + continue + p_dict = decode(rctx.read(path)) + p_name = p_dict.get("project", {}).get("name") + if p_name and canonicalize_name(p_name) == target_name: + return p_dict + + project_file = sibling_label if sibling_label in extra_project_files else extra_project_files[0] + project_path = rctx.path(project_file) + return decode(rctx.read(project_path)) if project_path.exists else {} + def read_project_and_lock(rctx, extra_project_files, lock_file, lock_model, lock_label_hint): """Select and decode pyproject.toml and the lock file. @@ -102,35 +125,7 @@ def read_project_and_lock(rctx, extra_project_files, lock_file, lock_model, lock Returns: A tuple of (project_dict, lock_dict). """ - projects = getattr(lock_model, "projects", []) - project_file = None - project_dict = None - - if not extra_project_files: - project_file = lock_file.relative(":pyproject.toml") - else: - if len(projects) == 1 and projects[0] != "*": - target_name = canonicalize_name(projects[0]) - for f in extra_project_files: - path = rctx.path(f) - if path.exists: - p_dict = decode(rctx.read(path)) - p_name = p_dict.get("project", {}).get("name") - if p_name and canonicalize_name(p_name) == target_name: - project_file = f - project_dict = p_dict - break - - if not project_file: - sibling_label = lock_file.relative(":pyproject.toml") - project_file = sibling_label if sibling_label in extra_project_files else extra_project_files[0] - - if project_dict == None: - project_dict = {} - if project_file: - project_path = rctx.path(project_file) - if project_path.exists: - project_dict = decode(rctx.read(project_path)) + project_dict = _read_project_dict(rctx, extra_project_files, lock_file, lock_model) lock_path = rctx.path(lock_file) if not lock_path.exists: @@ -241,6 +236,22 @@ def _version_key(version_str): """Parse a version string into a comparable key tuple.""" return pypackaging.version.parse(version_str).key +def _resolve_local_dep_constraint(constraint, dep_base_name, dep_version, resolution_marker_exprs): + """Combine a local package pin's constraint with any resolution-marker fork on its dependency.""" + cname = resolution_marker_constraint_name(dep_base_name, dep_version) + if cname not in resolution_marker_exprs or constraint == cname: + return constraint + if not constraint: + return cname + if type(resolution_marker_exprs.get(constraint)) == "string": + return constraint + dep_constraint = "{}_{}".format(constraint, cname) + resolution_marker_exprs[dep_constraint] = { + "variant": constraint, + "marker": cname, + } + return dep_constraint + def resolve_lock_graph(packages, pinned_package_specs, requires_python, strict_dependencies = True, variants = None, resolution_marker_exprs = None, testonly_pins = None, root_dependency_markers = None): """Resolves a dependency graph of packages. @@ -286,35 +297,36 @@ def resolve_lock_graph(packages, pinned_package_specs, requires_python, strict_d distinct_packages = {} for pkg in packages: pkg_key = _package_key_str(pkg["name"], pkg["version"]) - if pkg_key in distinct_packages: - # Merge: combine deps, files, extras - existing = distinct_packages[pkg_key] - merged_deps = list(existing["dependencies"]) - existing_dep_keys = {"{}/{}".format(d["name"], d.get("specifier", "")): True for d in merged_deps} - for d in pkg["dependencies"]: - dk = "{}/{}".format(d["name"], d.get("specifier", "")) - if dk not in existing_dep_keys: - merged_deps.append(d) - existing_dep_keys[dk] = True - merged_files = list(existing["files"]) - existing_file_names = {f["name"]: True for f in merged_files} - for f in pkg["files"]: - if f["name"] not in existing_file_names: - merged_files.append(f) - existing_file_names[f["name"]] = True - merged_extras = list(existing.get("extras", [])) - for e in pkg.get("extras", []): - if e not in merged_extras: - merged_extras.append(e) - distinct_packages[pkg_key] = dict( - existing, - dependencies = merged_deps, - files = merged_files, - extras = merged_extras, - is_local = existing.get("is_local", False) or pkg.get("is_local", False), - ) - else: + if pkg_key not in distinct_packages: distinct_packages[pkg_key] = pkg + continue + + # Merge: combine deps, files, extras + existing = distinct_packages[pkg_key] + merged_deps = list(existing["dependencies"]) + existing_dep_keys = {"{}/{}".format(d["name"], d.get("specifier", "")): True for d in merged_deps} + for d in pkg["dependencies"]: + dk = "{}/{}".format(d["name"], d.get("specifier", "")) + if dk not in existing_dep_keys: + merged_deps.append(d) + existing_dep_keys[dk] = True + merged_files = list(existing["files"]) + existing_file_names = {f["name"]: True for f in merged_files} + for f in pkg["files"]: + if f["name"] not in existing_file_names: + merged_files.append(f) + existing_file_names[f["name"]] = True + merged_extras = list(existing.get("extras", [])) + for e in pkg.get("extras", []): + if e not in merged_extras: + merged_extras.append(e) + distinct_packages[pkg_key] = dict( + existing, + dependencies = merged_deps, + files = merged_files, + extras = merged_extras, + is_local = existing.get("is_local", False) or pkg.get("is_local", False), + ) all_packages = distinct_packages.values() @@ -360,15 +372,16 @@ def resolve_lock_graph(packages, pinned_package_specs, requires_python, strict_d cand_extras = [canonicalize_name(e) for e in cand.get("extras", [])] if req_extra and canonicalize_name(req_extra) not in cand_extras: continue - if _specifier_contains_version(dep_specifier, cand["version"]): - pkg_resolved.append({ - "name": display_name, - "version": cand["version"], - "marker": dep_marker, - }) - found = True - if dep_specifier: - break + if not _specifier_contains_version(dep_specifier, cand["version"]): + continue + pkg_resolved.append({ + "name": display_name, + "version": cand["version"], + "marker": dep_marker, + }) + found = True + if dep_specifier: + break if not found and strict_dependencies: fail("Found no packages to satisfy dependency (name={}, spec={})".format(dep_name, dep_specifier)) @@ -390,10 +403,11 @@ def resolve_lock_graph(packages, pinned_package_specs, requires_python, strict_d for constraint, pin_specifier in pin_specs.items(): found = False for pin_pkg in pin_packages: - if _specifier_contains_version(pin_specifier, pin_pkg["version"]): - pinned_keys[pin_name][constraint] = _package_key_str(key_name or pin_pkg["name"], pin_pkg["version"]) - found = True - break + if not _specifier_contains_version(pin_specifier, pin_pkg["version"]): + continue + pinned_keys[pin_name][constraint] = _package_key_str(key_name or pin_pkg["name"], pin_pkg["version"]) + found = True + break if not found and strict_dependencies: fail("Found no packages to satisfy pin (name={}, spec={})".format(pin_name, pin_specifier)) @@ -428,17 +442,12 @@ def resolve_lock_graph(packages, pinned_package_specs, requires_python, strict_d dep_pin_name = dep_base_name # Keep resolution-marker forks of the dependency distinct. - dep_constraint = constraint - cname = resolution_marker_constraint_name(dep_base_name, dep["version"]) - if cname in resolution_marker_exprs and constraint != cname: - if not constraint: - dep_constraint = cname - elif type(resolution_marker_exprs.get(constraint)) != "string": - dep_constraint = "{}_{}".format(constraint, cname) - resolution_marker_exprs[dep_constraint] = { - "variant": constraint, - "marker": cname, - } + dep_constraint = _resolve_local_dep_constraint( + constraint, + dep_base_name, + dep["version"], + resolution_marker_exprs, + ) # The dependency is requested under the local package's edge # marker. A directly pinned name absent from diff --git a/pycross/private/uv_lock_model.bzl b/pycross/private/uv_lock_model.bzl index a894b6af..5f600300 100644 --- a/pycross/private/uv_lock_model.bzl +++ b/pycross/private/uv_lock_model.bzl @@ -84,24 +84,25 @@ def _resolve_package_requires_python(markers): for marker in markers: # Simple extraction of python_full_version constraints # e.g. "python_full_version >= '3.8'" -> ">= 3.8" - if "python_full_version" in marker: - # Parse out the operator and version - parts = marker.strip().split("python_full_version") - if len(parts) == 2: - rest = parts[1].strip() - - # Extract operator - op = "" - for c in rest.elems(): - if c in "<>=!": - op += c - elif c == " " and op: - break - elif op: - break - version_part = rest[len(op):].strip().strip("'\"") - if op and version_part: - specifiers.append("{} {}".format(op, version_part)) + if "python_full_version" not in marker: + continue + + # Parse out the operator and version + parts = marker.strip().split("python_full_version") + if len(parts) != 2: + continue + rest = parts[1].strip() + + # Extract operator + op = "" + for c in rest.elems(): + if c in "<>=!": + op += c + elif op: + break + version_part = rest[len(op):].strip().strip("'\"") + if op and version_part: + specifiers.append("{} {}".format(op, version_part)) return specifiers def _uv_conflict_token(item): @@ -136,32 +137,74 @@ def _parse_uv_dependency(dep, token_replacements = ()): Returns: A list of dep dicts suitable for resolve_lock_graph. """ - name = dep.get("name", "") + name = canonicalize_name(dep.get("name", "")) version = dep.get("version", "") + specifier = "==={}".format(version) if version else "" marker = _rewrite_conflict_markers(dep.get("marker", ""), token_replacements) extras = dep.get("extra") or dep.get("extras", []) - results = [] - if extras: - for extra in extras: - dep_name = "{}[{}]".format(canonicalize_name(name), canonicalize_name(extra)) - specifier = "==={}".format(version) if version else "" - results.append({ - "name": dep_name, - "specifier": specifier, - "marker": marker, - "extras": [None], - }) - else: - specifier = "==={}".format(version) if version else "" - results.append({ - "name": canonicalize_name(name), + if not extras: + return [{ + "name": name, "specifier": specifier, "marker": marker, "extras": [None], - }) + }] - return results + return [ + { + "name": "{}[{}]".format(name, canonicalize_name(extra)), + "specifier": specifier, + "marker": marker, + "extras": [None], + } + for extra in extras + ] + +def _resolve_uv_dep_constraint(constraint, base_dep_name, dep_version, dep_marker, fork_constraints, resolution_marker_exprs): + """Combine a group/extra variant constraint with any resolution-marker fork constraint.""" + if base_dep_name not in fork_constraints or dep_version not in fork_constraints[base_dep_name]: + return constraint + fork_constraint = fork_constraints[base_dep_name][dep_version] + if not constraint: + return fork_constraint + if dep_marker and dep_marker != resolution_marker_exprs.get(fork_constraint): + edge_fork_constraint = "{}__{}".format(fork_constraint, constraint) + resolution_marker_exprs[edge_fork_constraint] = dep_marker + fork_constraint = edge_fork_constraint + effective_constraint = "{}_{}".format(constraint, fork_constraint) + + # Register compound constraint with its components. + resolution_marker_exprs[effective_constraint] = { + "variant": constraint, + "marker": fork_constraint, + } + return effective_constraint + +def _parse_sdist_file(sdist, source, package_name, package_version, registry): + """Parse an sdist entry from a UV lock package, or return None.""" + if not sdist: + return None + if "url" not in sdist and "file" not in sdist and "url" in source: + # URL-based source with subdirectory + url = source["url"] + file_hash = sdist["hash"] + if not file_hash.startswith("sha256:"): + fail("Expected sha256: prefix on hash") + filename = "{}-{}{}".format(package_name, package_version, archive_extension(url)) + f = { + "name": filename, + "sha256": file_hash[7:], + "urls": [url], + "package_name": package_name, + "package_version": package_version, + } + if registry: + f["index"] = registry + return f + elif "git" not in source or "url" in sdist or "file" in sdist: + return _parse_file_info(sdist, package_name, package_version, registry) + return None def translate_uv(project_dict, lock_dict, lock_model): """Translates UV project and lock data to raw_lock_data dict. @@ -266,16 +309,16 @@ def translate_uv(project_dict, lock_dict, lock_model): target_projects = [canonicalize_name(p) for p in projects_list] for p in target_projects: + if p in workspace_members: + continue + + # Fallback for standalone single-project locks + for pkg in packages_list: + if canonicalize_name(pkg["name"]) == p: + workspace_members[p] = pkg + break if p not in workspace_members: - # Fallback for standalone single-project locks - found = False - for pkg in packages_list: - if canonicalize_name(pkg["name"]) == p: - workspace_members[p] = pkg - found = True - break - if not found: - fail("Project '{}' not found in uv.lock workspace members.".format(p)) + fail("Project '{}' not found in uv.lock workspace members.".format(p)) # Pre-scan: detect resolution-marker forks. # A fork exists when the same package name has multiple versions with @@ -345,12 +388,13 @@ def translate_uv(project_dict, lock_dict, lock_model): ]: for group_name, deps_list in groups_dict.items(): key = "{}:{}".format(kind, group_name) - if key in requested_groups_dict: - group_batches.append(( - constraint_dict.get((project_name, group_name), ""), - deps_list, - requested_groups_dict[key], - )) + if key not in requested_groups_dict: + continue + group_batches.append(( + constraint_dict.get((project_name, group_name), ""), + deps_list, + requested_groups_dict[key], + )) for constraint, deps_list, is_testonly in group_batches: for dep in deps_list: @@ -362,24 +406,14 @@ def translate_uv(project_dict, lock_dict, lock_model): # Resolution-marker fork: combine variant + fork constraints. base_dep_name = dep_name.split("[")[0] - if base_dep_name in _fork_constraints and dep_version in _fork_constraints[base_dep_name]: - fork_constraint = _fork_constraints[base_dep_name][dep_version] - if constraint: - if dep_marker and dep_marker != resolution_marker_exprs.get(fork_constraint): - edge_fork_constraint = "{}__{}".format(fork_constraint, constraint) - resolution_marker_exprs[edge_fork_constraint] = dep_marker - fork_constraint = edge_fork_constraint - effective_constraint = "{}_{}".format(constraint, fork_constraint) - - # Register compound constraint with its components. - resolution_marker_exprs[effective_constraint] = { - "variant": constraint, - "marker": fork_constraint, - } - else: - effective_constraint = fork_constraint - else: - effective_constraint = constraint + effective_constraint = _resolve_uv_dep_constraint( + constraint, + base_dep_name, + dep_version, + dep_marker, + _fork_constraints, + resolution_marker_exprs, + ) if dep_extras: for extra in dep_extras: @@ -438,27 +472,9 @@ def translate_uv(project_dict, lock_dict, lock_model): files.append(_parse_file_info(w, package_name, package_version, registry)) sdist = lock_pkg.get("sdist", {}) - - if sdist: - if "url" not in sdist and "file" not in sdist and "url" in source: - # URL-based source with subdirectory - url = source["url"] - file_hash = sdist["hash"] - if not file_hash.startswith("sha256:"): - fail("Expected sha256: prefix on hash") - filename = "{}-{}{}".format(package_name, package_version, archive_extension(url)) - f = { - "name": filename, - "sha256": file_hash[7:], - "urls": [url], - "package_name": package_name, - "package_version": package_version, - } - if registry: - f["index"] = registry - files.append(f) - elif "git" not in source or "url" in sdist or "file" in sdist: - files.append(_parse_file_info(sdist, package_name, package_version, registry)) + sdist_file = _parse_sdist_file(sdist, source, package_name, package_version, registry) + if sdist_file: + files.append(sdist_file) # Source dir source_dir = source.get("subdirectory", "") diff --git a/pycross/private/wheel_library.bzl b/pycross/private/wheel_library.bzl index b69e650d..ea3ad604 100644 --- a/pycross/private/wheel_library.bzl +++ b/pycross/private/wheel_library.bzl @@ -27,16 +27,7 @@ load(":util.bzl", "PY_COMMON_ATTRS", "merge_py_providers", "underscore_name") _PYCROSS_TOOLCHAIN_TYPE = Label("//pycross:toolchain_type") -def _pycross_wheel_library_impl(ctx): - out = ctx.actions.declare_directory(ctx.attr.name) - entry_points = ctx.actions.declare_file(ctx.attr.name + ".dist_info/entry_points.txt") - dist_info_dir = None - if ctx.attr.package_name and ctx.attr.package_version: - dist_info_dir = "{}-{}.dist-info".format( - underscore_name(ctx.attr.package_name), - ctx.attr.package_version, - ) - +def _register_wheel_install_action(ctx, out, entry_points, dist_info_dir): if PycrossUnsupportedWheelInfo in ctx.attr.wheel: pkg_id = "{}{}".format( ctx.attr.package_name or ctx.label.name, @@ -49,82 +40,171 @@ def _pycross_wheel_library_impl(ctx): mnemonic = "PycrossUnsupportedWheel", progress_message = "Rejecting unsupported wheel %s" % pkg_id, ) + return + + wheel_input = ctx.files.wheel[0] + + args = ctx.actions.args().use_param_file("--flagfile=%s") + if type(wheel_input) == "File" and wheel_input.is_directory: + args.add("--wheel-dir", wheel_input.path) else: - wheel_input = ctx.files.wheel[0] + # Plain file (e.g., local override wheel) — pass directly as --wheel + args.add("--wheel", wheel_input.path) + args.add("--directory", out.path) + args.add_all(ctx.files.post_install_patches, format_each = "--patch=%s") - args = ctx.actions.args().use_param_file("--flagfile=%s") - if type(wheel_input) == "File" and wheel_input.is_directory: - args.add("--wheel-dir", wheel_input.path) - else: - # Plain file (e.g., local override wheel) — pass directly as --wheel - args.add("--wheel", wheel_input.path) - args.add("--directory", out.path) - args.add_all(ctx.files.post_install_patches, format_each = "--patch=%s") - - inputs = [wheel_input] + ctx.files.post_install_patches - transitive_inputs = [] - tools = [] - - for install_exclude_glob in ctx.attr.install_exclude_globs: - args.add("--install-exclude-glob", install_exclude_glob) - - args.add("--entry-points-output", entry_points) - - if ctx.attr.package_name: - args.add("--expected-name", ctx.attr.package_name) - if ctx.attr.package_version: - args.add("--expected-version", ctx.attr.package_version) - if dist_info_dir: - args.add("--dist-info-dir", dist_info_dir) - - flag_precompile = ctx.attr._precompile[BuildSettingInfo].value - if flag_precompile == "force_enabled": - effective_precompile = "enabled" - elif flag_precompile == "force_disabled": - effective_precompile = "disabled" - elif ctx.attr.precompile == "auto": - effective_precompile = "enabled" if flag_precompile == "enabled" else "disabled" - else: - effective_precompile = ctx.attr.precompile - - if ( - effective_precompile == "enabled" and - _PYCROSS_TOOLCHAIN_TYPE in ctx.toolchains and - ctx.toolchains[_PYCROSS_TOOLCHAIN_TYPE] - ): - pycross_info = ctx.toolchains[_PYCROSS_TOOLCHAIN_TYPE].pycross_info - if ctx.attr.precompile_invalidation_mode == "auto": - invalidation_mode = ( - "unchecked_hash" if ctx.var.get("COMPILATION_MODE") == "opt" else "checked_hash" - ) - else: - invalidation_mode = ctx.attr.precompile_invalidation_mode - args.add("--compile-python", pycross_info.exec_python_executable) - args.add("--compile-invalidation-mode", invalidation_mode) - args.add("--compile-optimize", str(ctx.attr.precompile_optimize_level)) - compile_files = getattr( - pycross_info, - "exec_python_compile_files", - pycross_info.exec_python_files, + inputs = [wheel_input] + ctx.files.post_install_patches + transitive_inputs = [] + tools = [] + + for install_exclude_glob in ctx.attr.install_exclude_globs: + args.add("--install-exclude-glob", install_exclude_glob) + + args.add("--entry-points-output", entry_points) + + if ctx.attr.package_name: + args.add("--expected-name", ctx.attr.package_name) + if ctx.attr.package_version: + args.add("--expected-version", ctx.attr.package_version) + if dist_info_dir: + args.add("--dist-info-dir", dist_info_dir) + + flag_precompile = ctx.attr._precompile[BuildSettingInfo].value + if flag_precompile == "force_enabled": + effective_precompile = "enabled" + elif flag_precompile == "force_disabled": + effective_precompile = "disabled" + elif ctx.attr.precompile == "auto": + effective_precompile = "enabled" if flag_precompile == "enabled" else "disabled" + else: + effective_precompile = ctx.attr.precompile + + if ( + effective_precompile == "enabled" and + _PYCROSS_TOOLCHAIN_TYPE in ctx.toolchains and + ctx.toolchains[_PYCROSS_TOOLCHAIN_TYPE] + ): + pycross_info = ctx.toolchains[_PYCROSS_TOOLCHAIN_TYPE].pycross_info + if ctx.attr.precompile_invalidation_mode == "auto": + invalidation_mode = ( + "unchecked_hash" if ctx.var.get("COMPILATION_MODE") == "opt" else "checked_hash" ) - if compile_files: - transitive_inputs.append(compile_files) - if pycross_info.exec_python_files_to_run: - tools.append(pycross_info.exec_python_files_to_run) + else: + invalidation_mode = ctx.attr.precompile_invalidation_mode + args.add("--compile-python", pycross_info.exec_python_executable) + args.add("--compile-invalidation-mode", invalidation_mode) + args.add("--compile-optimize", str(ctx.attr.precompile_optimize_level)) + compile_files = getattr( + pycross_info, + "exec_python_compile_files", + pycross_info.exec_python_files, + ) + if compile_files: + transitive_inputs.append(compile_files) + if pycross_info.exec_python_files_to_run: + tools.append(pycross_info.exec_python_files_to_run) + + ctx.actions.run( + inputs = depset(inputs, transitive = transitive_inputs), + outputs = [out, entry_points], + executable = ctx.executable._tool, + tools = tools, + mnemonic = "PycrossWheelInstall", + arguments = [args], + env = { + "PYTHONHASHSEED": "0", + }, + progress_message = "Installing %s" % wheel_input.basename, + ) - ctx.actions.run( - inputs = depset(inputs, transitive = transitive_inputs), - outputs = [out, entry_points], - executable = ctx.executable._tool, - tools = tools, - mnemonic = "PycrossWheelInstall", - arguments = [args], - env = { - "PYTHONHASHSEED": "0", - }, - progress_message = "Installing %s" % wheel_input.basename, +def _build_venv_symlinks( + venvs_site_packages_enabled, + out, + imp, + dist_info_dir, + package_name, + package_version, + site_paths, + bin_paths, + data_paths, + include_paths): + if not (venvs_site_packages_enabled and site_paths): + return [] + + venv_symlinks = [] + for tlp in site_paths: + venv_symlinks.append(VenvSymlinkEntry( + kind = VenvSymlinkKind.LIB, + link_to_path = paths.join(imp, tlp), + package = package_name, + version = package_version, + venv_path = tlp, + files = depset([out]), + )) + + # Also add .dist-info directory symlink so metadata is accessible. + if dist_info_dir: + venv_symlinks.append(VenvSymlinkEntry( + kind = VenvSymlinkKind.LIB, + link_to_path = paths.join(imp, dist_info_dir), + package = package_name, + version = package_version, + venv_path = dist_info_dir, + files = depset([out]), + )) + + # Add other directory symlinks if supported by the rules_python version. + base_dir = paths.dirname(imp) + + if hasattr(VenvSymlinkKind, "BIN"): + for p in bin_paths: + venv_symlinks.append(VenvSymlinkEntry( + kind = getattr(VenvSymlinkKind, "BIN"), + link_to_path = paths.join(base_dir, "bin", p), + package = package_name, + version = package_version, + venv_path = p, + files = depset([out]), + )) + + if hasattr(VenvSymlinkKind, "DATA"): + for p in data_paths: + venv_symlinks.append(VenvSymlinkEntry( + kind = getattr(VenvSymlinkKind, "DATA"), + link_to_path = paths.join(base_dir, "data", p), + package = package_name, + version = package_version, + venv_path = p, + files = depset([out]), + )) + + # VenvSymlinkKind.INCLUDE existed in earlier versions of rules_python but was missing from venv_dir_map. + # It was properly added in the same commit that introduced VenvSymlinkKind.DATA. + if hasattr(VenvSymlinkKind, "INCLUDE"): + for p in include_paths: + venv_symlinks.append(VenvSymlinkEntry( + kind = getattr(VenvSymlinkKind, "INCLUDE"), + link_to_path = paths.join(base_dir, "include", p), + package = package_name, + version = package_version, + venv_path = p, + files = depset([out]), + )) + + return venv_symlinks + +def _pycross_wheel_library_impl(ctx): + out = ctx.actions.declare_directory(ctx.attr.name) + entry_points = ctx.actions.declare_file(ctx.attr.name + ".dist_info/entry_points.txt") + dist_info_dir = None + if ctx.attr.package_name and ctx.attr.package_version: + dist_info_dir = "{}-{}.dist-info".format( + underscore_name(ctx.attr.package_name), + ctx.attr.package_version, ) + _register_wheel_install_action(ctx, out, entry_points, dist_info_dir) + # TODO: Is there a more correct way to get this runfiles-relative import path? imp = paths.join( ctx.label.workspace_name or ctx.workspace_name, # Default to the local workspace. @@ -133,10 +213,6 @@ def _pycross_wheel_library_impl(ctx): "site-packages", # we put lib files in this subdirectory. ) - # Build venv symlink entries for rules_python's venvs_site_packages support. - # Each top-level package gets a symlink from the venv's site-packages/ - # to the runfiles location ///site-packages/. - venv_symlinks = [] package_name = ctx.attr.package_name or ctx.label.name package_version = ctx.attr.package_version or "" @@ -155,65 +231,21 @@ def _pycross_wheel_library_impl(ctx): venvs_site_packages_enabled = VenvsSitePackages.is_enabled(ctx) - if venvs_site_packages_enabled and site_paths: - for tlp in site_paths: - venv_symlinks.append(VenvSymlinkEntry( - kind = VenvSymlinkKind.LIB, - link_to_path = paths.join(imp, tlp), - package = package_name, - version = package_version, - venv_path = tlp, - files = depset([out]), - )) - - # Also add .dist-info directory symlink so metadata is accessible. - if dist_info_dir: - venv_symlinks.append(VenvSymlinkEntry( - kind = VenvSymlinkKind.LIB, - link_to_path = paths.join(imp, dist_info_dir), - package = package_name, - version = package_version, - venv_path = dist_info_dir, - files = depset([out]), - )) - - # Add other directory symlinks if supported by the rules_python version. - base_dir = paths.dirname(imp) - - if hasattr(VenvSymlinkKind, "BIN"): - for p in bin_paths: - venv_symlinks.append(VenvSymlinkEntry( - kind = getattr(VenvSymlinkKind, "BIN"), - link_to_path = paths.join(base_dir, "bin", p), - package = package_name, - version = package_version, - venv_path = p, - files = depset([out]), - )) - - if hasattr(VenvSymlinkKind, "DATA"): - for p in data_paths: - venv_symlinks.append(VenvSymlinkEntry( - kind = getattr(VenvSymlinkKind, "DATA"), - link_to_path = paths.join(base_dir, "data", p), - package = package_name, - version = package_version, - venv_path = p, - files = depset([out]), - )) - - # VenvSymlinkKind.INCLUDE existed in earlier versions of rules_python but was missing from venv_dir_map. - # It was properly added in the same commit that introduced VenvSymlinkKind.DATA. - if hasattr(VenvSymlinkKind, "INCLUDE"): - for p in include_paths: - venv_symlinks.append(VenvSymlinkEntry( - kind = getattr(VenvSymlinkKind, "INCLUDE"), - link_to_path = paths.join(base_dir, "include", p), - package = package_name, - version = package_version, - venv_path = p, - files = depset([out]), - )) + # Build venv symlink entries for rules_python's venvs_site_packages support. + # Each top-level package gets a symlink from the venv's site-packages/ + # to the runfiles location ///site-packages/. + venv_symlinks = _build_venv_symlinks( + venvs_site_packages_enabled, + out, + imp, + dist_info_dir, + package_name, + package_version, + site_paths, + bin_paths, + data_paths, + include_paths, + ) # With venvs_site_packages, the wheel is linked into the venv's site-packages, so (like rules_python) # don't also put its own site-packages dir on sys.path.