feat(rules): emit RunfilesGroupInfo from py_binary, py_test, and py_runtime

MODULE.bazel

@@ -9,6 +9,7 @@ bazel_dep(name = "bazel_skylib", version = "1.8.2")
 bazel_dep(name = "package_metadata", version = "0.0.7")
 bazel_dep(name = "platforms", version = "0.0.11")
 bazel_dep(name = "rules_cc", version = "0.2.17")
+bazel_dep(name = "rules_runfiles_group", version = "0.1.0")
 bazel_dep(name = "toml.bzl", version = "0.4.1")
 
 # Those are loaded only when using py_proto_library

docs/api/rules_python/python/config_settings/index.md

@@ -242,6 +242,55 @@ The `auto` value
 The `omit_if_generated_source` value was removed
 ::::
 
+::::{bzl:flag} runfiles_groups
+Determines if `py_binary`, `py_test`, and `py_runtime` emit the
+`RunfilesGroupInfo` provider from
+[`rules_runfiles_group`](https://github.com/bazel-contrib/rules_runfiles_group).
+
+:::{include} /_includes/experimental_api.md
+:::
+
+When enabled, an executable's runfiles are split into named groups, which
+packaging rules can consume to produce layered artifacts (e.g. container
+images with a shared interpreter layer and one layer per third-party
+dependency). The groups emitted by a binary are:
+
+* `rules_python#runtime`: the interpreter and its standard library.
+* `rules_python#pypi/<package>`: one group per PyPI-provided dependency.
+* One group per other `py_library` dependency, named by its label.
+* `rules_python#venv`: the binary-specific venv symlinks and support files.
+* `rules_python#app`: the binary's own code, executable, and bootstrap files.
+* One group per rule target in `data` attributes, named by its label.
+
+`py_library` participates through a private rules_python provider rather
+than by emitting `RunfilesGroupInfo` itself; its sources reach a binary via
+{obj}`PyInfo`, not through its own runfiles, so only a binary can produce
+groups whose union equals its `DefaultInfo.default_runfiles` (the public
+provider's contract). That exactness has two exceptions, in which only the
+coarse `runtime`/`venv`/`app` grouping is emitted: a binary that sets the
+{obj}`pyc_collection` attribute to a value different from the configuration
+default, and the deprecated implicit `__init__.py` creation.
+
+Values:
+
+* `auto`: (default) Follow the ecosystem-wide
+  `@rules_runfiles_group//runfiles_group:enabled` flag (itself default
+  disabled).
+* `disabled`: Don't emit the providers, regardless of the ecosystem-wide
+  flag.
+* `enabled`: Emit the providers, regardless of the ecosystem-wide flag.
+
+:::{note}
+Requires Bazel 9+. On older versions both flags are inert no-ops: Bazel 8's
+autoload machinery cannot load `rules_runfiles_group` from `rules_python`
+without a load cycle
+([bazel#23043](https://github.com/bazelbuild/bazel/issues/23043)).
+:::
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+::::
+
 ::::{bzl:flag} validate_test_main
 Determines if `py_test` runs a build-time validation that its main module
 actually runs tests.

news/3757.added.md

@@ -0,0 +1,10 @@
+(rules) Added experimental support for the `RunfilesGroupInfo` provider from
+[`rules_runfiles_group`](https://github.com/bazel-contrib/rules_runfiles_group).
+When enabled (via `--@rules_runfiles_group//runfiles_group:enabled` or
+{obj}`--@rules_python//python/config_settings:runfiles_groups`), `py_binary`
+and `py_test` split their runfiles into named groups (interpreter plus
+stdlib, one group per dependency, the binary-specific venv, and the binary's
+own code) so packaging rules can build layered container images with a
+shared interpreter layer and one layer per package
+([#3757](https://github.com/bazel-contrib/rules_python/issues/3757)).
+Requires Bazel 9+.

python/config_settings/BUILD.bazel

@@ -10,6 +10,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "RunfilesGroupsFlag",
     "ValidateTestMainFlag",
     "VenvsSitePackages",
     "VenvsUseDeclareSymlinkFlag",
@@ -79,6 +80,14 @@ string_flag(
     visibility = NOT_ACTUALLY_PUBLIC,
 )
 
+string_flag(
+    name = "runfiles_groups",
+    build_setting_default = RunfilesGroupsFlag.AUTO,
+    values = RunfilesGroupsFlag.flag_values(),
+    # NOTE: Only public because it is dependency of public rules.
+    visibility = NOT_ACTUALLY_PUBLIC,
+)
+
 string_flag(
     name = "validate_test_main",
     build_setting_default = ValidateTestMainFlag.AUTO,

python/private/BUILD.bazel

@@ -564,6 +564,7 @@ bzl_library(
         ":py_runtime_info",
         ":reexports",
         ":rule_builders",
+        ":runfiles_groups",
         ":toolchain_types",
         ":transition_labels",
         ":venv_runfiles",
@@ -616,6 +617,7 @@ bzl_library(
         ":py_info",
         ":reexports",
         ":rule_builders",
+        ":runfiles_groups",
         ":toolchain_types",
         ":venv_runfiles",
         ":version",
@@ -690,13 +692,27 @@ bzl_library(
         ":py_internal",
         ":py_runtime_info",
         ":reexports",
+        ":runfiles_groups",
         ":version",
         "@bazel_skylib//lib:dicts",
         "@bazel_skylib//lib:paths",
         "@bazel_skylib//rules:common_settings",
     ],
 )
 
+bzl_library(
+    name = "runfiles_groups",
+    srcs = ["runfiles_groups.bzl"],
+    deps = [
+        ":flags",
+        ":py_info",
+        ":py_internal",
+        ":reexports",
+        "@bazel_skylib//rules:common_settings",
+        "@rules_python_internal//:runfiles_groups_shim",
+    ],
+)
+
 bzl_library(
     name = "py_test_macro",
     srcs = ["py_test_macro.bzl"],

python/private/common_labels.bzl

@@ -30,6 +30,7 @@ labels = struct(
     PY_FREETHREADED = str(Label("//python/config_settings:py_freethreaded")),
     PY_LINUX_LIBC = str(Label("//python/config_settings:py_linux_libc")),
     REPL_DEP = str(Label("//python/bin:repl_dep")),
+    RUNFILES_GROUPS = str(Label("//python/config_settings:runfiles_groups")),
     VALIDATE_TEST_MAIN = str(Label("//python/config_settings:validate_test_main")),
     VENV = str(Label("//python/config_settings:venv")),
     VENVS_SITE_PACKAGES = str(Label("//python/config_settings:venvs_site_packages")),

python/private/flags.bzl

@@ -87,6 +87,20 @@ AddSrcsToRunfilesFlag = FlagEnum(
     is_enabled = _AddSrcsToRunfilesFlag_is_enabled,
 )
 
+# Determines if py_library/py_binary/py_test emit runfiles group providers.
+# The effective value also depends on the ecosystem-wide
+# @rules_runfiles_group//runfiles_group:enabled switch; the combination is
+# resolved by is_enabled() in runfiles_groups.bzl, which is the only reader.
+# buildifier: disable=name-conventions
+RunfilesGroupsFlag = FlagEnum(
+    # Follow the @rules_runfiles_group//runfiles_group:enabled flag.
+    AUTO = "auto",
+    # Emit the providers, regardless of the ecosystem-wide flag.
+    ENABLED = "enabled",
+    # Don't emit the providers, regardless of the ecosystem-wide flag.
+    DISABLED = "disabled",
+)
+
 def _ValidateTestMainFlag_is_enabled(ctx):
     value = ctx.attr._validate_test_main_flag[BuildSettingInfo].value
     if value == ValidateTestMainFlag.AUTO:

python/private/internal_config_repo.bzl

@@ -40,6 +40,28 @@ config = struct(
 )
 """
 
+# rules_runfiles_group isn't in Bazel 8's hardcoded list of repositories that
+# are exempt from autoloads, so loading its bzl files from rules_python creates
+# a load cycle there (https://github.com/bazelbuild/bazel/issues/23043). This
+# repo is on that list, so the load is routed through here and stubbed out on
+# Bazel versions with autoloads.
+_RUNFILES_GROUPS_SHIM_TEMPLATE = """
+load("@rules_runfiles_group//runfiles_group:lib.bzl", _runfiles_groups = "runfiles_groups")
+load("@rules_runfiles_group//runfiles_group:providers.bzl", _RunfilesGroupInfo = "RunfilesGroupInfo")
+
+RUNFILES_GROUPS_AVAILABLE = True
+runfiles_groups = _runfiles_groups
+RunfilesGroupInfo = _RunfilesGroupInfo
+ENABLED_FLAG_LABEL = "@rules_runfiles_group//runfiles_group:enabled"
+"""
+
+_RUNFILES_GROUPS_SHIM_UNAVAILABLE_TEMPLATE = """
+RUNFILES_GROUPS_AVAILABLE = False
+runfiles_groups = None
+RunfilesGroupInfo = None
+ENABLED_FLAG_LABEL = "//python:none"
+"""
+
 ROOT_BUILD_TEMPLATE = """
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 
@@ -58,6 +80,12 @@ bzl_library(
     name = "rules_python_config",
     srcs = ["rules_python_config.bzl"],
 )
+
+bzl_library(
+    name = "runfiles_groups_shim",
+    srcs = ["runfiles_groups_shim.bzl"],
+    deps = [{runfiles_groups_shim_deps}],
+)
 """
 
 _EXTRA_TRANSITIONS_TEMPLATE = """
@@ -114,8 +142,17 @@ def _internal_config_repo_impl(rctx):
         bazel_10_or_later = str(bazel_major_version > 9),
     ))
 
+    if bazel_major_version >= 9:
+        runfiles_groups_shim = _RUNFILES_GROUPS_SHIM_TEMPLATE
+        runfiles_groups_shim_deps = '"@rules_runfiles_group//runfiles_group:lib", "@rules_runfiles_group//runfiles_group:providers"'
+    else:
+        runfiles_groups_shim = _RUNFILES_GROUPS_SHIM_UNAVAILABLE_TEMPLATE
+        runfiles_groups_shim_deps = ""
+    rctx.file("runfiles_groups_shim.bzl", runfiles_groups_shim)
+
     rctx.file("BUILD", ROOT_BUILD_TEMPLATE.format(
         visibility = "@rules_python//:__subpackages__",
+        runfiles_groups_shim_deps = runfiles_groups_shim_deps,
     ))
 
     rctx.file(

python/private/py_executable.bzl

@@ -68,6 +68,21 @@ load(":py_interpreter_program.bzl", "PyInterpreterProgramInfo")
 load(":py_runtime_info.bzl", "DEFAULT_STUB_SHEBANG")
 load(":reexports.bzl", "BuiltinPyInfo", "BuiltinPyRuntimeInfo")
 load(":rule_builders.bzl", "ruleb")
+load(
+    ":runfiles_groups.bzl",
+    "APP_GROUP",
+    "RUNFILES_GROUP_ENABLED_LABEL",
+    "app_entry",
+    "build_entries_depset",
+    "collect_data_entries",
+    "collect_dep_entries",
+    "collect_src_entries",
+    "create_runfiles_group_info",
+    "pyc_collection_enabled_by_default",
+    "runtime_entry",
+    "venv_entry",
+    runfiles_groups_enabled = "is_enabled",
+)
 load(":toolchain_types.bzl", "CC_TOOLCHAIN_TYPE", "EXEC_TOOLS_TOOLCHAIN_TYPE", "LAUNCHER_MAKER_TOOLCHAIN_TYPE", TOOLCHAIN_TYPE = "TARGET_TOOLCHAIN_TYPE")
 load(":transition_labels.bzl", "TRANSITION_LABELS")
 load(":venv_runfiles.bzl", "create_venv_app_files")
@@ -91,6 +106,8 @@ EXECUTABLE_ATTRS = dicts.add(
         "_default_to_explicit_init_py_flag": attr.label(default = "//python/config_settings:incompatible_default_to_explicit_init_py"),
         "_python_import_all_repositories_flag": attr.label(default = "//python/config_settings:experimental_python_import_all_repositories"),
         "_python_path_flag": attr.label(default = "//python/config_settings:python_path"),
+        "_runfiles_group_enabled": attr.label(default = RUNFILES_GROUP_ENABLED_LABEL),
+        "_runfiles_groups_flag": attr.label(default = labels.RUNFILES_GROUPS),
     },
     {
         "interpreter_args": lambda: attrb.StringList(
@@ -502,6 +519,11 @@ WARNING: Target: {}
         # depset[ExplicitSymlink]None; symlinks that should be created in
         # the venv to augment app_runfiles
         venv_app_symlinks = venv.lib_symlinks if venv else None,
+        # runfiles|None; the non-interpreter venv files: site-packages
+        # symlinks and small generated files (pth, site init, pyvenv.cfg).
+        venv_files_runfiles = (
+            ctx.runfiles(venv.files_without_interpreter).merge(venv.lib_runfiles) if venv else None
+        ),
         # File|None; the venv `bin/python3` file, if any.
         venv_python_exe = venv.interpreter if venv else None,
         # runfiles|None; runfiles in the venv for the interpreter
@@ -1305,8 +1327,119 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment =
         _maybe_add_test_main_validation(ctx, main_py_source, output_groups)
     _add_provider_output_group_info(providers, py_info, output_groups)
 
+    if runfiles_groups_enabled(ctx):
+        providers.append(_create_runfiles_groups(
+            ctx,
+            executable = executable,
+            runtime_details = runtime_details,
+            runfiles_details = runfiles_details,
+            exec_result = exec_result,
+            cc_details = cc_details,
+            native_deps_details = native_deps_details,
+            extra_deps = extra_deps,
+            required_py_files = required_py_files,
+            required_pyc_files = required_pyc_files,
+            implicit_pyc_files = implicit_pyc_files,
+            implicit_pyc_source_files = implicit_pyc_source_files,
+        ))
+
     return providers
 
+def _create_runfiles_groups(
+        ctx,
+        *,
+        executable,
+        runtime_details,
+        runfiles_details,
+        exec_result,
+        cc_details,
+        native_deps_details,
+        extra_deps,
+        required_py_files,
+        required_pyc_files,
+        implicit_pyc_files,
+        implicit_pyc_source_files):
+    """Creates the public RunfilesGroupInfo provider.
+
+    The union of all groups equals `DefaultInfo.default_runfiles` exactly;
+    packaging rules depend on that invariant. See runfiles_groups.bzl for
+    how the groups fit together.
+    """
+    pyc_collection_enabled = PycCollectionAttr.is_pyc_collection_enabled(ctx)
+
+    # Dependencies compute their entries against the configuration
+    # (see pyc_collection_enabled_by_default). If this binary's
+    # `pyc_collection` attribute overrides that, the dependency-provided
+    # entries don't match what this binary adds to its runfiles, so fall
+    # back to coarse grouping. The same applies to the deprecated implicit
+    # `__init__.py` creation, whose synthesized empty files only exist in
+    # `app_runfiles`.
+    fine_grained = (
+        pyc_collection_enabled == pyc_collection_enabled_by_default(ctx) and
+        not _should_create_init_files(ctx)
+    )
+
+    collected = []
+    if fine_grained:
+        dep_entries = collect_dep_entries(
+            ctx,
+            ctx.attr.deps + extra_deps,
+            pyc_collection_enabled = pyc_collection_enabled,
+        )
+        src_entries = collect_src_entries(ctx.attr.srcs)
+        data_entries = collect_data_entries(ctx.attr.data)
+        collected = [dep_entries, src_entries, data_entries]
+
+        app_files = builders.DepsetBuilder()
+        app_files.add(required_py_files)
+        app_files.add(required_pyc_files)
+        if pyc_collection_enabled:
+            app_files.add(implicit_pyc_files)
+        else:
+            app_files.add(implicit_pyc_source_files)
+        app_files.add(data_entries.own_files)
+        app_files.add(exec_result.extra_default_outputs)
+        app_files.add(executable)
+        if exec_result.stage2_bootstrap:
+            app_files.add(exec_result.stage2_bootstrap)
+        if runfiles_details.build_data_file:
+            app_files.add(runfiles_details.build_data_file)
+        app_runfiles = ctx.runfiles(transitive_files = app_files.build())
+        app_runfiles = app_runfiles.merge_all([
+            cc_details.extra_runfiles,
+            native_deps_details.runfiles,
+        ])
+    else:
+        # `app_runfiles` is everything except the runtime and venv:
+        # the binary's own sources, all dependencies, and data.
+        coarse_files = builders.DepsetBuilder()
+        coarse_files.add(exec_result.extra_default_outputs)
+        coarse_files.add(executable)
+        if exec_result.stage2_bootstrap:
+            coarse_files.add(exec_result.stage2_bootstrap)
+        if runfiles_details.build_data_file:
+            coarse_files.add(runfiles_details.build_data_file)
+        app_runfiles = ctx.runfiles(transitive_files = coarse_files.build())
+        app_runfiles = app_runfiles.merge(runfiles_details.app_runfiles)
+
+    direct = [app_entry(app_runfiles)]
+
+    runtime_runfiles = runtime_details.runfiles
+    if exec_result.venv_interpreter_runfiles:
+        runtime_runfiles = runtime_runfiles.merge(
+            exec_result.venv_interpreter_runfiles,
+        )
+    direct.append(runtime_entry(runtime_runfiles))
+
+    if exec_result.venv_files_runfiles:
+        direct.append(venv_entry(exec_result.venv_files_runfiles))
+
+    collected.append(struct(direct = direct, transitive = []))
+    return create_runfiles_group_info(
+        build_entries_depset(*collected),
+        executable_group = APP_GROUP,
+    )
+
 def _maybe_add_test_main_validation(ctx, main_py, output_groups):
     """Adds a validation action that checks the test main actually runs tests.