diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 928ad1072..353d6dfa8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -89,7 +89,7 @@ jobs: - name: 'Install Ruff' run: | if ! command -v ruff &>/dev/null; then - pip install --no-cache-dir ruff + pip install --no-cache-dir ruff==0.16.4 fi - name: 'Run Ruff Lint Check' @@ -101,7 +101,7 @@ jobs: - name: 'Install Mypy' run: | if ! command -v mypy &>/dev/null; then - pip install --no-cache-dir mypy + pip install --no-cache-dir mypy==2.4.1 fi - name: 'Run Mypy Static Type Check' @@ -169,6 +169,12 @@ jobs: run: | python3 tools/lint/check_agent_doc_paths.py + - name: 'Check Documentation Links & OSS URLs' + env: + XMC_BYPASS_VENV: '1' + run: | + python3 tools/lint/check_doc_links.py + - name: 'Check UI & Workflow File Naming' env: XMC_BYPASS_VENV: '1' @@ -355,9 +361,9 @@ jobs: - name: 'Install Helm' if: steps.changes.outputs.helm == 'true' run: | - if ! command -v helm &>/dev/null; then + if ! helm version --short 2>/dev/null | grep -q '^v3\.'; then mkdir -p "${RUNNER_TEMP}/helm" - curl -sSfL "https://get.helm.sh/helm-v3.15.2-linux-amd64.tar.gz" \ + curl -sSfL "https://get.helm.sh/helm-v3.20.1-linux-amd64.tar.gz" \ | tar -xz -C "${RUNNER_TEMP}/helm" --strip-components=1 linux-amd64/helm chmod +x "${RUNNER_TEMP}/helm/helm" echo "${RUNNER_TEMP}/helm" >> "${GITHUB_PATH}" @@ -380,3 +386,17 @@ jobs: severity: 'HIGH,CRITICAL' scan-ref: 'deploy/' trivyignores: '.trivyignore' + + # 8. PR Diff Size Telemetry + - name: 'Check PR Diff Size' + if: github.event_name == 'pull_request' + env: + XMC_BYPASS_VENV: '1' + BASE_REF: ${{ github.base_ref }} + run: | + DIFF_JSON="$(python3 tools/analyze_diff_size.py --base-ref "origin/${BASE_REF}...")" + echo "${DIFF_JSON}" + if [ "$(echo "${DIFF_JSON}" | jq -r '.result.threshold_exceeded')" = 'true' ]; then + REASONS="$(echo "${DIFF_JSON}" | jq -r '.result.exceeded_reasons | join("; ")')" + echo "::warning::PR diff size exceeds review threshold: ${REASONS}" + fi diff --git a/tools/README.md b/tools/README.md index fb9230a49..656154682 100644 --- a/tools/README.md +++ b/tools/README.md @@ -84,8 +84,9 @@ Tooling and automation for release metadata extraction, SemVer calculations, and ### 7. `tools/lint/` (Static Analyzers & Linters) Offline syntax walkers and static validation tooling: - [`check_go_determinism`](lint/check_go_determinism/): Ultra-fast (<50ms) Go AST analyzer checking Go test files for `os.Setenv` usage and arbitrary second-level `time.Sleep` calls. -- [`check_legacy_imports.py`](lint/check_legacy_imports.py): Ratchet lint preventing regression of google3 migration leftovers (legacy Go import roots, unresolved `xmanager_cloud/...` module paths, `compat_go_*` targets, `gazelle:resolve` overrides, `third_party/` proto paths, google3 Helm flags, legacy UI/Python paths) against the zeroed [`legacy_imports_baseline.json`](lint/legacy_imports_baseline.json). Fails when any count grows above zero. Runs in pre-commit and the lint CI job. Tracking: [#1066](https://github.com/google/xmc/issues/1066). +- [`check_legacy_imports.py`](lint/check_legacy_imports.py): Ratchet lint preventing regression of google3 migration leftovers (legacy Go import roots, unresolved `xmanager_cloud/...` module paths, `compat_go_*` targets, `gazelle:resolve` overrides, `third_party/` proto paths, google3 Helm flags, legacy UI/Python paths), OSS `TODO(b/...)` IDs under `api/`, `cmd/`, `pkg/`, and `sdks/`, and new generic helper/util/misc/common filenames against [`legacy_imports_baseline.json`](lint/legacy_imports_baseline.json). Fails when any count grows above baseline. Runs in pre-commit and the lint CI job. Tracking: [#1066](https://github.com/google/xmc/issues/1066). - [`check_agent_doc_paths.py`](lint/check_agent_doc_paths.py): Fails when agent docs (`.agents/**/*.md`, `AGENTS.md`) reference repo paths in code spans or fenced blocks that are not tracked files or directories. Placeholders (``, `{a,b}`, `$VAR`, `*`, `[`, `...`) only require their directory prefix to exist; runtime-created and illustrative paths live in its `ALLOWLIST` with reasons, and unreferenced entries fail. Runs in pre-commit (when agent docs are staged or files are deleted/renamed; always in `--check` mode) and the lint CI job. +- [`check_doc_links.py`](lint/check_doc_links.py): Validates relative Markdown links (`[label](path)`) against tracked files and directories and rejects non-code `file://`, `corp.google.com`, `http(s)://go/...`, and `//go/...` URLs across all tracked Markdown docs outside `.agents/`. Runs in pre-commit (when non-`.agents/` Markdown docs or doc linters are staged, or files are deleted/renamed; always in `--check` mode) and the lint CI job. - [`check_file_naming.py`](lint/check_file_naming.py): Enforces UI and workflow file-naming conventions across tracked repository files (`*.ng.html` for HTML files under `cmd/ui/`—templates under `cmd/ui/app/` with `cmd/ui/index.html` allowlisted—`*_test.ts` for Karma unit tests under `cmd/ui/app/` while rejecting `*.spec.ts`/`*_spec.ts`/`*.test.ts` repo-wide outside `cmd/ui/screenshot_tests/`, `*.screenshot.spec.ts` for Playwright visual specs under `cmd/ui/screenshot_tests/`, and top-level `*.test.js` for Node workflow tests under `.github/scripts/`). Runs in pre-commit and the lint CI job. - [`check_ui_selector_contract.py`](lint/check_ui_selector_contract.py): Parses every tracked `*.py` under `tools/nightly_tests/` (fails on any syntax error; `EXCLUDED_SOURCES` opts files out) and lints each file containing selectors (`*_SEL`/`*_SEL_TMPL` constants, `.locator()`/`.wait_for_selector()` string arguments, or `has_text=` literals; reported as `sources`). Fails when selectors reference `[data-testid="..."]` values or custom tags that no `cmd/ui/app` template or component selector declares (Material `mat-`/`mdc-`/`cdk-` names exempt), when selectors use custom classes or non-standard attributes without `# ui-contract: `, when code references a `ui_css_selectors` constant that does not exist, or when selectors match user-facing copy via `:has-text(...)`/`has_text=` without a `# ui-contract: ` comment on the same line or a standalone comment line above. Runs on every commit in pre-commit and in the lint CI job (~0.2s; no path gate, so the source list is not duplicated). - [`precommit_gates.py`](lint/precommit_gates.py): Declarative runner (`--mode fix|check`) and shared git/execution primitives for pre-commit Python lint gates (`check_legacy_imports`, `check_examples_coverage`, `check_matrix_contract`, `check_agent_doc_paths`, `check_file_naming`, `check_ui_selector_contract`). diff --git a/tools/lint/BUILD.bazel b/tools/lint/BUILD.bazel index 64fe37db5..7cc3b86f8 100644 --- a/tools/lint/BUILD.bazel +++ b/tools/lint/BUILD.bazel @@ -37,6 +37,27 @@ py_test( ], ) +py_library( + name = "check_doc_links_lib", + srcs = ["check_doc_links.py"], + imports = ["../.."], + deps = [ + ":check_agent_doc_paths_lib", + "//tools/common:tool_utils", + ], +) + +py_test( + name = "check_doc_links_test", + srcs = ["check_doc_links_test.py"], + imports = ["../.."], + deps = [ + ":check_agent_doc_paths_lib", + ":check_doc_links_lib", + "//tools/common:tool_utils", + ], +) + py_library( name = "check_file_naming_lib", srcs = ["check_file_naming.py"], diff --git a/tools/lint/check_agent_doc_paths.py b/tools/lint/check_agent_doc_paths.py index 25137260f..ab0181e2c 100644 --- a/tools/lint/check_agent_doc_paths.py +++ b/tools/lint/check_agent_doc_paths.py @@ -108,8 +108,8 @@ def build_path_token_re(roots: Iterable[str]) -> re.Pattern[str]: _PATH_TOKEN_RE = build_path_token_re(PATH_ROOTS) -_INLINE_CODE_RE = re.compile(r"(`+)(.+?)\1") -_FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$") +INLINE_CODE_RE = re.compile(r"(? bool: ) +def opens_fence(line: str) -> str: + """Returns the opening fence marker when a line opens a CommonMark fenced block.""" + fence = _FENCE_RE.match(line) + if not fence: + return "" + return fence.group(1) or fence.group(3) + + def closes_fence(line: str, opening: str) -> bool: """Returns whether a line closes a fenced block opened with `opening`. @@ -142,7 +150,8 @@ def closes_fence(line: str, opening: str) -> bool: fence = _FENCE_RE.match(line) if not fence: return False - run, rest = fence.group(1), fence.group(2) + run = fence.group(1) or fence.group(3) + rest = fence.group(2) if fence.group(1) is not None else fence.group(4) return run[0] == opening[0] and len(run) >= len(opening) and not rest.strip() @@ -156,11 +165,11 @@ def iter_code_segments(text: str) -> Iterator[tuple[int, str]]: else: yield lineno, line continue - fence = _FENCE_RE.match(line) - if fence: - opening = fence.group(1) + opened = opens_fence(line) + if opened: + opening = opened continue - for match in _INLINE_CODE_RE.finditer(line): + for match in INLINE_CODE_RE.finditer(line): yield lineno, match.group(2) diff --git a/tools/lint/check_doc_links.py b/tools/lint/check_doc_links.py new file mode 100644 index 000000000..c47e9a60d --- /dev/null +++ b/tools/lint/check_doc_links.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lint verifying relative links and OSS URL hygiene in repository Markdown docs. + +Scans tracked `*.md` files outside `.agents/` (which `check_agent_doc_paths.py` +checks separately) and fails when: +- A relative Markdown link target does not exist in the git index or escapes the repo. +- A non-code Markdown line contains a `file://` URI, `corp.google.com` host, or + `http(s)://go/...` / `//go/...` corporate shortlink. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import posixpath +import re +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any +from urllib.parse import unquote + +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from tools.common.tool_utils import HandledError, run_tool # noqa: E402 +from tools.lint.check_agent_doc_paths import ( # noqa: E402 + INLINE_CODE_RE, + closes_fence, + list_tracked_files, + opens_fence, + tracked_paths, +) + +_LINK_TAIL_RE = re.compile(r'^\s*(?:(?:"[^"]*"|\'[^\']*\')\s*)?\)') +_REF_DEF_RE = re.compile(r"^\s{0,3}\[[^\]]+\]:\s*(<[^>]+>|[^\s]+)") +_SCHEME_RE = re.compile(r"^[a-z][a-z0-9+.-]+:", re.IGNORECASE) +_FORBIDDEN_URL_RE = re.compile( + r"file://[^\s)>'\"]+|[A-Za-z0-9.-]*corp\.google\.com\b|(?:https?://|(? bool: + """Return whether a tracked path is a Markdown doc covered by this linter.""" + return path.endswith(".md") and not path.startswith(".agents/") + + +def iter_prose_lines(text: str) -> Iterator[tuple[int, str]]: + """Yield (1-based line number, line without inline code) outside fenced blocks.""" + opening = "" + for lineno, line in enumerate(text.splitlines(), start=1): + if opening: + if closes_fence(line, opening): + opening = "" + continue + opened = opens_fence(line) + if opened: + opening = opened + continue + yield lineno, INLINE_CODE_RE.sub("", line) + + +def _parse_link_destination(prose: str, start: int) -> str | None: + """Parse an inline Markdown link destination immediately following `](`.""" + i = start + while i < len(prose) and prose[i] in " \t": + i += 1 + if i < len(prose) and prose[i] == "<": + end = prose.find(">", i + 1) + if end == -1 or not _LINK_TAIL_RE.match(prose[end + 1 :]): + return None + return prose[i + 1 : end] + depth = 0 + j = i + while j < len(prose): + ch = prose[j] + if ch in " \t": + break + if ch == "(": + depth += 1 + elif ch == ")": + if depth == 0: + return prose[i:j] if j > i else None + depth -= 1 + j += 1 + if j > i and depth == 0 and _LINK_TAIL_RE.match(prose[j:]): + return prose[i:j] + return None + + +def extract_link_targets(prose: str) -> list[str]: + """Extract inline link, linked-image, and reference-definition targets from a line.""" + targets: list[str] = [] + pos = 0 + while (idx := prose.find("](", pos)) != -1: + dest = _parse_link_destination(prose, idx + 2) + if dest is not None: + targets.append(dest) + pos = idx + 2 + ref_match = _REF_DEF_RE.match(prose) + if ref_match: + targets.append(ref_match.group(1)) + return targets + + +def resolve_repo_target(doc: str, raw_target: str) -> tuple[str, str | None]: + """Resolve a Markdown link target relative to doc. + + Returns: + Tuple of (resolved_repo_path, error_kind) where error_kind is None if + the link is external, anchor-only, or resolves cleanly within the repo. + """ + stripped = raw_target.strip("<>") + if not stripped or stripped.startswith("#"): + return "", None + if stripped.lower().startswith("file://"): + return stripped.rstrip(".,;:"), "forbidden_url" + if stripped.startswith("//") or _SCHEME_RE.match(stripped): + return "", None + path_part = unquote(stripped.split("#", 1)[0].split("?", 1)[0]) + if not path_part: + return "", None + if path_part.startswith("/"): + joined = posixpath.normpath(path_part.lstrip("/")) + else: + doc_dir = posixpath.dirname(doc) + joined = posixpath.normpath(posixpath.join(doc_dir, path_part) if doc_dir else path_part) + if joined == ".." or joined.startswith("../"): + return joined, "escapes_repo" + return ("" if joined == "." else joined), None + + +def find_line_violations( + doc: str, lineno: int, prose: str, existing: frozenset[str] +) -> list[DocViolation]: + """Collect broken-link and forbidden-URL violations from a single prose line.""" + violations: list[DocViolation] = [] + for raw_target in extract_link_targets(prose): + resolved, err_kind = resolve_repo_target(doc, raw_target) + if err_kind is not None: + violations.append(DocViolation(doc=doc, line=lineno, kind=err_kind, target=resolved)) + elif resolved and resolved not in existing: + violations.append( + DocViolation(doc=doc, line=lineno, kind="missing_target", target=resolved) + ) + for url_match in _FORBIDDEN_URL_RE.finditer(prose): + url = url_match.group(0).rstrip(".,;:") + if not any(v.target == url and v.line == lineno for v in violations): + violations.append(DocViolation(doc=doc, line=lineno, kind="forbidden_url", target=url)) + return violations + + +def scan_docs(repo_root: Path, docs: list[str], existing: frozenset[str]) -> list[DocViolation]: + """Scan all tracked Markdown docs outside `.agents/` for link and URL issues.""" + violations: list[DocViolation] = [] + for doc in docs: + doc_path = repo_root / doc + if not doc_path.is_file(): + continue + text = doc_path.read_text(encoding="utf-8", errors="replace") + for lineno, prose in iter_prose_lines(text): + violations.extend(find_line_violations(doc, lineno, prose, existing)) + return violations + + +def check_doc_links(args: argparse.Namespace) -> dict[str, Any]: + """Validate relative links and public URL hygiene in repository Markdown docs.""" + repo_root = Path(args.repo_root).resolve() + files = list_tracked_files(repo_root) + docs = sorted(f for f in files if is_checked_doc(f)) + violations = scan_docs(repo_root, docs, tracked_paths(files)) + if violations: + raise HandledError( + f"Found {len(violations)} broken Markdown link(s) or internal URL(s).", + "Fix broken relative Markdown links against tracked files/directories and remove " + "local file:// or corporate URLs.", + details={"violations": [dataclasses.asdict(v) for v in violations]}, + ) + return {"docs_scanned": len(docs), "violations": 0} + + +def main() -> None: + """CLI entrypoint.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--repo-root", default=str(_REPO_ROOT), help="Repository root.") + run_tool(parser, check_doc_links) + + +if __name__ == "__main__": + main() diff --git a/tools/lint/check_doc_links_test.py b/tools/lint/check_doc_links_test.py new file mode 100644 index 000000000..310cd795a --- /dev/null +++ b/tools/lint/check_doc_links_test.py @@ -0,0 +1,258 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for check_doc_links.""" + +from __future__ import annotations + +import argparse +import io +import json +import sys +import tempfile +import unittest +from contextlib import redirect_stderr +from pathlib import Path +from unittest import mock + +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from tools.common.tool_utils import HandledError # noqa: E402 +from tools.lint import check_doc_links as cli # noqa: E402 +from tools.lint.check_agent_doc_paths import tracked_paths # noqa: E402 + +_EXISTING = tracked_paths( + [ + "README.md", + "docs/developer.md", + "docs/my file.md", + "docs/foo(1).md", + "deploy/ctk_v2/README.md", + ] +) + + +class ResolveRepoTargetTest(unittest.TestCase): + def test_ignores_external_and_anchor_links(self) -> None: + self.assertEqual(cli.resolve_repo_target("docs/a.md", "https://example.com"), ("", None)) + self.assertEqual(cli.resolve_repo_target("docs/a.md", "HTTPS://example.com"), ("", None)) + self.assertEqual(cli.resolve_repo_target("docs/a.md", "http://example.com"), ("", None)) + self.assertEqual(cli.resolve_repo_target("docs/a.md", "mailto:x@example.com"), ("", None)) + self.assertEqual(cli.resolve_repo_target("docs/a.md", "MAILTO:x@example.com"), ("", None)) + self.assertEqual(cli.resolve_repo_target("docs/a.md", "gs://bucket/path"), ("", None)) + self.assertEqual(cli.resolve_repo_target("docs/a.md", "//example.com/path"), ("", None)) + self.assertEqual(cli.resolve_repo_target("docs/a.md", "#section"), ("", None)) + + def test_resolves_relative_links_with_fragments(self) -> None: + self.assertEqual( + cli.resolve_repo_target("docs/design/topologies.md", "../../deploy/ctk_v2/README.md#s"), + ("deploy/ctk_v2/README.md", None), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "../README.md?plain=1"), + ("README.md", None), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "/docs/developer.md"), + ("docs/developer.md", None), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "my%20file.md"), + ("docs/my file.md", None), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "my%20file.md?plain=1#section"), + ("docs/my file.md", None), + ) + + def test_flags_repo_escape_and_file_uri(self) -> None: + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "../../outside.md"), + ("../outside.md", "escapes_repo"), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "/../../outside.md"), + ("../../outside.md", "escapes_repo"), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "%2e%2e/%2e%2e/outside.md"), + ("../outside.md", "escapes_repo"), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "file:///tmp/x.md"), + ("file:///tmp/x.md", "forbidden_url"), + ) + self.assertEqual( + cli.resolve_repo_target("docs/a.md", "FILE:///tmp/x.md"), + ("FILE:///tmp/x.md", "forbidden_url"), + ) + + +class FindLineViolationsTest(unittest.TestCase): + def test_allows_existing_targets_and_code_examples(self) -> None: + text = ( + "See [Dev Guide](developer.md#setup), [Ümlaut](/docs/developer.md), and `file:///tmp/example`.\n" + "````md\n" + "```md\n" + "[missing](nonexistent.md) http://go/example\n" + "```\n" + "````\n" + "~~~~bash\n" + "[tilde-nested](nonexistent.md)\n" + "~~~~\n" + ) + found = [] + for lineno, prose in cli.iter_prose_lines(text): + found.extend(cli.find_line_violations("docs/a.md", lineno, prose, _EXISTING)) + self.assertEqual(found, []) + + def test_handles_empty_malformed_unclosed_fence_and_multibyte(self) -> None: + self.assertEqual(list(cli.iter_prose_lines("")), []) + malformed = "[unclosed(target.md) and 日本語 [broken](missing-日本語.md)\n" + malformed_found = [] + for lineno, prose in cli.iter_prose_lines(malformed): + malformed_found.extend(cli.find_line_violations("docs/a.md", lineno, prose, _EXISTING)) + self.assertEqual( + [(v.kind, v.target) for v in malformed_found], + [("missing_target", "docs/missing-日本語.md")], + ) + unclosed_fence = "```\n[ignored](nope.md)\n" + self.assertEqual(list(cli.iter_prose_lines(unclosed_fence)), []) + + def test_handles_nested_links_parens_spaces_refs_and_fence_edges(self) -> None: + valid = ( + "[![alt](developer.md)](/docs/foo(1).md) " + "[space]( 'title') " + "[ext](//example.com/a) " + "[gcs](gs://my-bucket/obj)\n" + "[ref]: /docs/developer.md\n" + ) + valid_found = [] + for lineno, prose in cli.iter_prose_lines(valid): + valid_found.extend(cli.find_line_violations("docs/a.md", lineno, prose, _EXISTING)) + self.assertEqual(valid_found, []) + + tricky = ( + "Use ```file:///tmp/x``` and [broken1](m1.md)\n" + "Use ``unclosed [broken2](m2.md) and `ok`\n" + "Use ``foo``` and [broken3](m3.md) and `bar`\n" + "[![alt](missing_img.png)](missing_outer.md)\n" + "[trailing](file:///tmp/x.md.)\n" + "[badref]: ../missing_ref.md\n" + ) + tricky_found = [] + for lineno, prose in cli.iter_prose_lines(tricky): + tricky_found.extend(cli.find_line_violations("docs/a.md", lineno, prose, _EXISTING)) + self.assertEqual( + [(v.line, v.kind, v.target) for v in tricky_found], + [ + (1, "missing_target", "docs/m1.md"), + (2, "missing_target", "docs/m2.md"), + (3, "missing_target", "docs/m3.md"), + (4, "missing_target", "docs/missing_img.png"), + (4, "missing_target", "docs/missing_outer.md"), + (5, "forbidden_url", "file:///tmp/x.md"), + (6, "missing_target", "missing_ref.md"), + ], + ) + + def test_flags_missing_relative_links_and_internal_urls(self) -> None: + text = ( + "Old [link](missing.md) and [corp](https://moma.corp.google.com/foo) " + "plus http://go/internal-doc, [proto](//go/proto-doc), " + "https://example.com/a//go/safe, and file:///etc/passwd.\n" + ) + found = [] + for lineno, prose in cli.iter_prose_lines(text): + found.extend(cli.find_line_violations("docs/a.md", lineno, prose, _EXISTING)) + self.assertEqual( + [(v.kind, v.target) for v in found], + [ + ("missing_target", "docs/missing.md"), + ("forbidden_url", "moma.corp.google.com"), + ("forbidden_url", "http://go/internal-doc"), + ("forbidden_url", "//go/proto-doc"), + ("forbidden_url", "file:///etc/passwd"), + ], + ) + + def test_ignores_agents_docs(self) -> None: + self.assertFalse(cli.is_checked_doc(".agents/rules/markdown.md")) + self.assertTrue(cli.is_checked_doc("docs/developer.md")) + self.assertTrue(cli.is_checked_doc("README.md")) + + +class CheckDocLinksCliTest(unittest.TestCase): + def test_scan_passes_and_fails_cleanly(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "docs").mkdir() + (root / "README.md").write_text("# Root\n", encoding="utf-8") + (root / "docs/ok.md").write_text("See [Root](../README.md).\n", encoding="utf-8") + (root / "docs/bad.md").write_text("See [Missing](nope.md).\n", encoding="utf-8") + + with mock.patch.object( + cli, "list_tracked_files", return_value=["README.md", "docs/ok.md"] + ): + res = cli.check_doc_links(argparse.Namespace(repo_root=tmp)) + self.assertEqual(res, {"docs_scanned": 2, "violations": 0}) + + with mock.patch.object( + cli, "list_tracked_files", return_value=["README.md", "docs/bad.md"] + ): + with self.assertRaises(HandledError) as ctx: + cli.check_doc_links(argparse.Namespace(repo_root=tmp)) + assert ctx.exception.details is not None + self.assertEqual( + ctx.exception.details["violations"], + [ + { + "doc": "docs/bad.md", + "line": 1, + "kind": "missing_target", + "target": "docs/nope.md", + } + ], + ) + + # Working-tree deletion not yet staged is skipped gracefully; repo escape fails. + (root / "docs/esc.md").write_text("See [Out](../../out.md).\n", encoding="utf-8") + with mock.patch.object( + cli, + "list_tracked_files", + return_value=["README.md", "docs/deleted.md", "docs/esc.md"], + ): + with self.assertRaises(HandledError) as esc_ctx: + cli.check_doc_links(argparse.Namespace(repo_root=tmp)) + assert esc_ctx.exception.details is not None + self.assertEqual(esc_ctx.exception.details["violations"][0]["kind"], "escapes_repo") + + def test_main_outputs_json_and_exits_one_on_violation(self) -> None: + buf = io.StringIO() + err = HandledError("bad link", "fix it", details={"violations": []}) + with ( + mock.patch.object(sys, "argv", ["check_doc_links.py", "--repo-root", "/tmp"]), + mock.patch.object(cli, "check_doc_links", side_effect=err), + redirect_stderr(buf), + self.assertRaises(SystemExit) as exit_ctx, + ): + cli.main() + self.assertEqual(exit_ctx.exception.code, 1) + self.assertEqual(json.loads(buf.getvalue())["status"], "error") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/lint/check_legacy_imports.py b/tools/lint/check_legacy_imports.py index 75b2ed341..4cf9747e8 100644 --- a/tools/lint/check_legacy_imports.py +++ b/tools/lint/check_legacy_imports.py @@ -28,8 +28,10 @@ import argparse import dataclasses import json +import posixpath import re import sys +from collections.abc import Iterable from pathlib import Path from typing import Any @@ -118,8 +120,27 @@ class LineMetric: path_regex=r"\.(py|sh)$", line_regex=r"third_party[./](xmanager_cloud|javascript)|_virtual_imports/.*/third_party", ), + LineMetric( + name="oss_b_todo_references", + path_regex=r"^(api|cmd|pkg|sdks)/", + line_regex=r"TODO\([^)]*\bb/\d+", + ), ) +_GENERIC_FILENAME_RE = re.compile(r"(?:util|helper|misc|common)", re.IGNORECASE) + + +def is_generic_helper_filename(path: str) -> bool: + """Reports whether path uses a generic filename banned for new modules.""" + if path.startswith("deploy/") and "/templates/" in path and path.endswith(".tpl"): + return False + return bool(_GENERIC_FILENAME_RE.search(posixpath.basename(path))) + + +def count_generic_helper_filenames(files: Iterable[str]) -> int: + """Counts tracked files whose basenames use generic helper/util/misc/common names.""" + return sum(1 for f in files if is_generic_helper_filename(f)) + def extract_go_imports(source: str) -> list[str]: """Returns import paths declared in a Go source file.""" @@ -199,6 +220,7 @@ def collect_metrics(repo_root: Path, files: list[str]) -> dict[str, int]: scanned = [f for f in files if f not in SELF_EXCLUDED] metrics = count_go_imports(repo_root, scanned, tracked_directories(files)) metrics["third_party_files"] = sum(1 for f in scanned if f.startswith("third_party/")) + metrics["generic_helper_filenames"] = count_generic_helper_filenames(scanned) for metric in LINE_METRICS: metrics[metric.name] = count_line_metric(metric, repo_root, scanned) return dict(sorted(metrics.items())) @@ -277,7 +299,9 @@ def check_legacy_imports(args: argparse.Namespace) -> dict[str, Any]: if diff["regressed"]: raise HandledError( f"google3 legacy constructs increased: {json.dumps(diff['regressed'])}", - "Use OSS import paths and standard Bazel rules; see github.com/google/xmc/issues/1066.", + "Use OSS import paths, OSS TODOs (no TODO(b/...)), descriptive filenames " + "(avoid util/helper/misc/common), and standard Bazel rules; see " + "github.com/google/xmc/issues/1066.", ) return { "metrics": current, diff --git a/tools/lint/check_legacy_imports_test.py b/tools/lint/check_legacy_imports_test.py index fd9c580fe..b5d5fa099 100644 --- a/tools/lint/check_legacy_imports_test.py +++ b/tools/lint/check_legacy_imports_test.py @@ -99,6 +99,27 @@ def test_tracked_directories(self): frozenset({"a", "a/b"}), ) + def test_is_generic_helper_filename_boundaries(self): + self.assertTrue(cli.is_generic_helper_filename("pkg/urlUtil.go")) + self.assertTrue(cli.is_generic_helper_filename("tools/misc_ops.py")) + self.assertTrue(cli.is_generic_helper_filename("pkg/common.go")) + self.assertTrue(cli.is_generic_helper_filename("deploy/helper.tpl")) + self.assertTrue(cli.is_generic_helper_filename("deploy/helm/templates/helper.yaml")) + self.assertTrue(cli.is_generic_helper_filename("pkg/templates/_helpers.tpl")) + self.assertFalse(cli.is_generic_helper_filename("deploy/helm/templates/_helpers.tpl")) + + def test_oss_b_todo_references_scoping(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for rel in ("api/a.proto", "cmd/a.go", "pkg/a.go", "internal/a.go", "tools/a.py"): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("// TODO(b/12345): fix\n") + metrics = cli.collect_metrics( + root, ["api/a.proto", "cmd/a.go", "pkg/a.go", "internal/a.go", "tools/a.py"] + ) + self.assertEqual(metrics["oss_b_todo_references"], 3) + class CollectMetricsTest(unittest.TestCase): def setUp(self): @@ -121,7 +142,9 @@ def setUp(self): "deploy/ctk_v2/m/main.tf": '"-alsologtostderr",\n"-log_keyword=x",\n', "deploy/helm/templates/e.yaml": "args: ['--notracelog']\ncommand: run -vmodule=x\n", "cmd/ui/app/a.ts": "import { X } from 'javascript/closure/format/format';\n", - "sdks/p.py": "from third_party.xmanager_cloud.x import y\n", + "sdks/p.py": "from third_party.xmanager_cloud.x import y\n# TODO(b/12345): fix\n", + "pkg/helper.go": "package helper\n", + "deploy/helm/templates/_helpers.tpl": '{{- define "xmc" -}}{{- end -}}\n', "third_party/grpc_gateway.BUILD": "", # Self-excluded files must not be counted. "tools/lint/check_legacy_imports_test.py": "from third_party.xmanager_cloud import x\n", @@ -141,9 +164,11 @@ def test_collects_all_metrics(self): { "compat_go_targets": 2, "gazelle_resolve_directives": 1, + "generic_helper_filenames": 1, "go_legacy_root_imports": 3, "go_unresolved_module_imports": 1, "helm_google3_flags": 6, + "oss_b_todo_references": 1, "proto_third_party_paths": 2, "python_legacy_paths": 1, "third_party_files": 1, diff --git a/tools/lint/legacy_imports_baseline.json b/tools/lint/legacy_imports_baseline.json index 62075b113..69e184fdd 100644 --- a/tools/lint/legacy_imports_baseline.json +++ b/tools/lint/legacy_imports_baseline.json @@ -1,9 +1,11 @@ { "compat_go_targets": 0, "gazelle_resolve_directives": 0, + "generic_helper_filenames": 28, "go_legacy_root_imports": 0, "go_unresolved_module_imports": 0, "helm_google3_flags": 0, + "oss_b_todo_references": 0, "proto_third_party_paths": 0, "python_legacy_paths": 0, "third_party_files": 0, diff --git a/tools/lint/precommit_gates.py b/tools/lint/precommit_gates.py index 9d973d31f..cde72a9b2 100644 --- a/tools/lint/precommit_gates.py +++ b/tools/lint/precommit_gates.py @@ -64,7 +64,17 @@ class PythonGate: banner="--> Checking Agent Doc Path References...", script="tools/lint/check_agent_doc_paths.py", failure_msg="Agent docs reference non-existent repository paths (see JSON error above).", - staged_am_pattern=r"^\.agents/.*\.md$|(^|/)AGENTS\.md$", + staged_am_pattern=r"^\.agents/.*\.md$|(^|/)AGENTS\.md$|^tools/lint/check_agent_doc_paths\.py$", + check_deletes=True, + ), + PythonGate( + banner="--> Checking Documentation Links & OSS URLs...", + script="tools/lint/check_doc_links.py", + failure_msg=( + "Markdown docs contain broken relative links or forbidden internal URLs " + "(see JSON error above)." + ), + staged_am_pattern=r"\.md$|^tools/lint/check_(doc_links|agent_doc_paths)\.py$", check_deletes=True, ), PythonGate( diff --git a/tools/lint/precommit_gates_test.py b/tools/lint/precommit_gates_test.py index 239495d59..b1be6d01f 100644 --- a/tools/lint/precommit_gates_test.py +++ b/tools/lint/precommit_gates_test.py @@ -468,6 +468,19 @@ def _fail_first(ctx: gates.RunContext, _gate: gates.PythonGate) -> None: ): self.assertEqual(gates.main(["--mode", "fix"]), 1) + def test_python_gates_include_doc_link_and_agent_doc_triggers(self): + by_script = {g.script: g for g in gates.PYTHON_GATES} + self.assertIn("tools/lint/check_doc_links.py", by_script) + self.assertEqual( + by_script["tools/lint/check_doc_links.py"].staged_am_pattern, + r"\.md$|^tools/lint/check_(doc_links|agent_doc_paths)\.py$", + ) + self.assertTrue(by_script["tools/lint/check_doc_links.py"].check_deletes) + self.assertEqual( + by_script["tools/lint/check_agent_doc_paths.py"].staged_am_pattern, + r"^\.agents/.*\.md$|(^|/)AGENTS\.md$|^tools/lint/check_agent_doc_paths\.py$", + ) + if __name__ == "__main__": unittest.main()