Infer dependencies across local replace directives in the Go backend (closes #22097)

docs/notes/2.33.x.md

@@ -87,6 +87,8 @@ Fixes JavaScript workspace builds where dependencies are installed under members
 
 Third-party module analysis is now deduplicated across `go.mod` files. Previously, a module required by `N` `go.mod` files was downloaded and analyzed `N` times, which caused significant memory and time overhead in monorepos with many overlapping `go.mod` files. On a 3-`go.mod` reproducer, `pants list ::` peak memory dropped from 91 GB to 32 GB (-65%). This is a no-op for repos with a single `go.mod`. See [#20274](https://github.com/pantsbuild/pants/issues/20274).
 
+Dependency inference now resolves imports across local directory `replace` directives. When a `go.mod` replaces a required module with a path to another first-party module in the repo, imports of that module's packages are inferred automatically instead of having to be listed by hand in BUILD metadata. This is a no-op for modules without directory `replace` directives. See [#22097](https://github.com/pantsbuild/pants/issues/22097).
+
 ### Plugin API changes
 
 `FrozenOrderedSet` is now backed by a native Rust implementation and is built in `__new__` rather than `__init__`. Subclasses that customized the set's contents by overriding `__init__` (for example, to transform the input iterable before constructing) must move that logic to `__new__`, since the set is already built and immutable by the time `__init__` runs.

src/python/pants/backend/go/target_type_rules.py

@@ -39,6 +39,7 @@
     GoModInfoRequest,
     OwningGoModRequest,
     determine_go_mod_info,
+    find_all_go_mod_targets,
     find_owning_go_mod,
 )
 from pants.backend.go.util_rules.import_analysis import (
@@ -210,7 +211,52 @@ async def map_import_paths_to_packages(
     request: GoImportPathMappingRequest,
     module_import_path_mappings: AllGoModuleImportPathsMappings,
 ) -> GoModuleImportPathsMapping:
-    return module_import_path_mappings.modules[request.go_mod_address]
+    base = module_import_path_mappings.modules[request.go_mod_address]
+
+    # #22097: a `go.mod` may `replace` a `require`d module with a local directory that holds
+    # another first-party `go_mod` target. Imports of that module's packages are not in this
+    # module's own import-path map, so fold in the replaced modules' maps. Go's `replace`
+    # directives only apply within the module that declares them (they are not transitive), so a
+    # single level of merging matches the build semantics.
+    go_mod_info = await determine_go_mod_info(GoModInfoRequest(request.go_mod_address))
+    if not go_mod_info.local_replace_module_dirs:
+        return base
+
+    all_go_mod_targets = await find_all_go_mod_targets(**implicitly())
+    address_by_spec_path = {tgt.address.spec_path: tgt.address for tgt in all_go_mod_targets}
+
+    merged_mapping: dict[str, GoImportPathsMappingAddressSet] = dict(base.mapping)
+    merged_address_to_import_path: dict[Address, str] = dict(base.address_to_import_path)
+    for replaced_module_path, replaced_dir in go_mod_info.local_replace_module_dirs.items():
+        replaced_address = address_by_spec_path.get(replaced_dir)
+        if replaced_address is None or replaced_address == request.go_mod_address:
+            continue
+        replaced_mapping = module_import_path_mappings.modules.get(replaced_address)
+        if replaced_mapping is None:
+            continue
+        # Only fold in the replaced module's OWN packages -- import paths under its module path.
+        # Its third-party dependencies must NOT be merged: the referencing module resolves those
+        # through its own `go.mod`, and merging them would make every shared third-party import
+        # path ambiguous (two addresses), which suppresses the inferred dependency.
+        prefix = f"{replaced_module_path}/"
+        for import_path, address_set in replaced_mapping.mapping.items():
+            if import_path != replaced_module_path and not import_path.startswith(prefix):
+                continue
+            existing = merged_mapping.get(import_path)
+            if existing is None:
+                merged_mapping[import_path] = address_set
+            else:
+                merged_mapping[import_path] = GoImportPathsMappingAddressSet(
+                    addresses=tuple(sorted({*existing.addresses, *address_set.addresses})),
+                    infer_all=existing.infer_all or address_set.infer_all,
+                )
+            for address in address_set.addresses:
+                merged_address_to_import_path.setdefault(address, import_path)
+
+    return GoModuleImportPathsMapping(
+        mapping=FrozenDict(merged_mapping),
+        address_to_import_path=FrozenDict(merged_address_to_import_path),
+    )
 
 
 @dataclass(frozen=True)

src/python/pants/backend/go/target_type_rules_test.py

@@ -161,6 +161,147 @@ def get_deps(addr: Address) -> set[Address]:
     assert not get_deps(Address("foo/bad"))
 
 
+def test_cross_module_local_replace_dependency_inference(rule_runner: RuleRunner) -> None:
+    # A package in one module that imports a package from a sibling first-party module, wired via a
+    # local-directory `replace` directive, should have that dependency inferred (#22097).
+    rule_runner.write_files(
+        {
+            "app/BUILD": "go_mod()",
+            "app/go.mod": dedent(
+                """\
+                module go.example.com/app
+                go 1.17
+
+                require go.example.com/lib v0.0.0
+
+                replace go.example.com/lib => ../lib
+                """
+            ),
+            "app/go.sum": "",
+            "app/pkg/app.go": dedent(
+                """\
+                package pkg
+
+                import "go.example.com/lib/greet"
+
+                var _ = greet.Hello
+                """
+            ),
+            "app/pkg/BUILD": "go_package()",
+            "lib/BUILD": "go_mod()",
+            "lib/go.mod": dedent(
+                """\
+                module go.example.com/lib
+                go 1.17
+                """
+            ),
+            "lib/greet/greet.go": dedent(
+                """\
+                package greet
+
+                const Hello = "hello"
+                """
+            ),
+            "lib/greet/BUILD": "go_package()",
+        }
+    )
+
+    def get_deps(addr: Address) -> set[Address]:
+        tgt = rule_runner.get_target(addr)
+        return set(
+            rule_runner.request(
+                Addresses,
+                [DependenciesRequest(tgt[Dependencies])],
+            )
+        )
+
+    assert get_deps(Address("app/pkg")) == {Address("lib/greet")}
+
+
+def test_cross_module_local_replace_does_not_shadow_third_party(rule_runner: RuleRunner) -> None:
+    # A locally-replaced module's OWN third-party dependencies must not be folded into the
+    # referencing module's import map. If they were, a third-party import shared by both modules
+    # would map to two addresses (one generated per `go_mod`), become ambiguous, and have its
+    # inferred dependency silently dropped (#22097 regression guard).
+    go_sum = dedent(
+        """\
+        golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8=
+        golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+        rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
+        rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+        rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
+        rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+        """
+    )
+    rule_runner.write_files(
+        {
+            "app/BUILD": "go_mod()",
+            "app/go.mod": dedent(
+                """\
+                module go.example.com/app
+                go 1.17
+
+                require (
+                    go.example.com/lib v0.0.0
+                    rsc.io/quote v1.5.2
+                )
+
+                replace go.example.com/lib => ../lib
+                """
+            ),
+            "app/go.sum": go_sum,
+            "app/pkg/app.go": dedent(
+                """\
+                package pkg
+
+                import (
+                    "go.example.com/lib/greet"
+                    "rsc.io/quote"
+                )
+
+                var _ = greet.Hello
+                var _ = quote.Hello
+                """
+            ),
+            "app/pkg/BUILD": "go_package()",
+            "lib/BUILD": "go_mod()",
+            "lib/go.mod": dedent(
+                """\
+                module go.example.com/lib
+                go 1.17
+
+                require rsc.io/quote v1.5.2
+                """
+            ),
+            "lib/go.sum": go_sum,
+            "lib/greet/greet.go": dedent(
+                """\
+                package greet
+
+                import "rsc.io/quote"
+
+                var Hello = quote.Hello
+                """
+            ),
+            "lib/greet/BUILD": "go_package()",
+        }
+    )
+
+    def get_deps(addr: Address) -> set[Address]:
+        tgt = rule_runner.get_target(addr)
+        return set(
+            rule_runner.request(
+                Addresses,
+                [DependenciesRequest(tgt[Dependencies])],
+            )
+        )
+
+    deps = get_deps(Address("app/pkg"))
+    # The first-party cross-module import resolves, AND the shared third-party import resolves to
+    # `app`'s own generated target -- unambiguously, not dropped.
+    assert deps == {Address("lib/greet"), Address("app", generated_name="rsc.io/quote")}
+
+
 # -----------------------------------------------------------------------------------------------
 # `go_package` validation
 # -----------------------------------------------------------------------------------------------

src/python/pants/backend/go/util_rules/go_mod.py

@@ -38,6 +38,7 @@
     WrappedTargetRequest,
 )
 from pants.util.docutil import bin_name
+from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 
 logger = logging.getLogger(__name__)
@@ -215,13 +216,29 @@ async def find_owning_go_mod(
     )
 
 
+def _is_directory_replacement(new_path: str) -> bool:
+    # Mirrors `modfile.IsDirectoryPath`: a `replace` directive whose target is a local
+    # directory (as opposed to a different module path) starts with `./`, `../`, `/`, or
+    # is `.`/`..` (or an absolute Windows path). Such replacements wire a `require`d module
+    # to a sibling first-party module on disk; module-path replacements do not.
+    return (
+        new_path in (".", "..")
+        or new_path.startswith(("./", "../", "/"))
+        or os.path.isabs(new_path)
+    )
+
+
 @dataclass(frozen=True)
 class GoModInfo:
     # Import path of the Go module, based on the `module` in `go.mod`.
     import_path: str
     digest: Digest
     mod_path: str
     minimum_go_version: str | None
+    # Maps the module path of each `replace`d module whose target is a local directory to
+    # that directory (relative to the build root). These are first-party modules referenced
+    # across `go.mod` boundaries; see `map_import_paths_to_packages` and issue #22097.
+    local_replace_module_dirs: FrozenDict[str, str] = FrozenDict()
 
 
 @dataclass(frozen=True)
@@ -265,11 +282,29 @@ async def determine_go_mod_info(
         )
     )
     module_metadata = json.loads(mod_json.stdout)
+
+    # Record `replace` directives that point a `require`d module at a local directory, so that
+    # dependency inference can resolve imports satisfied by sibling first-party modules. See #22097.
+    local_replace_module_dirs: dict[str, str] = {}
+    for replace in module_metadata.get("Replace") or []:
+        new = replace.get("New") or {}
+        new_path = new.get("Path")
+        # A directory replacement targets a filesystem path and carries no module version; a
+        # module-path replacement (path + version) is left to normal third-party resolution.
+        if not new_path or new.get("Version") or not _is_directory_replacement(new_path):
+            continue
+        old_path = (replace.get("Old") or {}).get("Path")
+        # Absolute on-disk paths cannot be mapped to in-repo `go_mod` targets, so skip them.
+        if not old_path or os.path.isabs(new_path):
+            continue
+        local_replace_module_dirs[old_path] = os.path.normpath(os.path.join(go_mod_dir, new_path))
+
     return GoModInfo(
         import_path=module_metadata["Module"]["Path"],
         digest=sources_digest,
         mod_path=go_mod_path,
         minimum_go_version=module_metadata.get("Go"),
+        local_replace_module_dirs=FrozenDict(local_replace_module_dirs),
     )
 
 

src/python/pants/backend/go/util_rules/third_party_pkg.py

@@ -251,8 +251,9 @@ async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> Modu
         if "Main" in mod_json and mod_json["Main"]:
             continue
 
-        # Skip first-party modules referenced from other first-party modules.
-        # TODO Issue #22097: These cross-module references could be used for dependency inference
+        # Skip first-party modules referenced from other first-party modules via a local
+        # directory `replace`. These are not third-party packages; `map_import_paths_to_packages`
+        # folds their import paths into the referencing module's inference map (#22097).
         if "Replace" in mod_json and "Version" not in mod_json["Replace"]:
             continue