diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml
index 2afb1eebb7d..21aa7edfe92 100644
--- a/.github/workflows/mvn-verify-check.yml
+++ b/.github/workflows/mvn-verify-check.yml
@@ -430,3 +430,23 @@ jobs:
fi
}
done
+
+ with-resource-nesting-audit:
+ name: withResource nesting audit
+ runs-on: ubuntu-latest
+ steps:
+ - uses: NVIDIA/spark-rapids-common/checkout@main
+
+ - name: Report withResource nesting violations
+ run: |
+ mvn --batch-mode -N antrun:run@with-resource-nesting-audit \
+ -DwithResource.audit.rawReport="$RUNNER_TEMP/with-resource-nesting-audit.json" \
+ -DwithResource.audit.summary="$GITHUB_STEP_SUMMARY"
+
+ - name: Upload withResource nesting report
+ if: ${{ always() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: with-resource-nesting-audit
+ path: ${{ runner.temp }}/with-resource-nesting-audit.json
+ if-no-files-found: ignore
diff --git a/pom.xml b/pom.xml
index ac0296ce52a..45cf4840373 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1153,6 +1153,8 @@
4.9.10
3.1.1
3.3.0
+ ${project.build.directory}/with-resource-nesting-audit.json
+ ${project.build.directory}/with-resource-nesting-summary.md
2.0.2
30.0-jre
2.0.0
@@ -1993,6 +1995,37 @@ This will force full Scala code rebuild in downstream modules.
false
+
+
+
+ with-resource-nesting-audit
+ verify
+ run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml
index a9d3442ef2b..c0540689dfc 100644
--- a/scala2.13/pom.xml
+++ b/scala2.13/pom.xml
@@ -1153,6 +1153,8 @@
4.9.10
3.1.1
3.3.0
+ ${project.build.directory}/with-resource-nesting-audit.json
+ ${project.build.directory}/with-resource-nesting-summary.md
2.0.2
30.0-jre
2.0.0
@@ -1993,6 +1995,37 @@ This will force full Scala code rebuild in downstream modules.
false
+
+
-
-
-
-
-
diff --git a/scripts/README.md b/scripts/README.md
index b8c49752346..00aa34a78eb 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -51,4 +51,11 @@ Only hoist values that own their data. Views returned by methods such as `bitCas
`getChildColumnView`, `replaceListChild`, and `splitAsViews` must not outlive their owning parent.
Copy or convert a view to an owning resource before closing its parent.
-The check and its unit tests run with the all-modules Scalastyle execution during `mvn verify`.
+The script remains directly runnable with Python 3, but is also compatible with Jython 2.7. Maven
+uses its managed Jython dependency to run the check and unit tests during root `mvn verify`; the
+generated Scala 2.13 reactor does not repeat this repository-wide check.
+
+The pull-request audit publishes every deep scope in the job summary and a JSON artifact. Existing
+baseline entries are reported as debt but do not fail the check. New violations, stale baseline
+entries, invalid exemptions, or report-generation errors fail it. GitHub source annotations show
+the first 50 entries, with the complete set retained in the summary and artifact.
diff --git a/scripts/check_with_resource_nesting.py b/scripts/check_with_resource_nesting.py
index 4cbc3a2ed38..b198e13a715 100644
--- a/scripts/check_with_resource_nesting.py
+++ b/scripts/check_with_resource_nesting.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
# Copyright (c) 2026, NVIDIA CORPORATION.
#
@@ -16,17 +16,16 @@
"""Prevent new deeply nested withResource scopes in production Scala code."""
-from __future__ import annotations
+from __future__ import print_function
import argparse
import collections
-import dataclasses
import hashlib
+import io
import json
+import os
import re
import sys
-from pathlib import Path
-from typing import Iterable, Sequence
DEFAULT_MAX_DEPTH = 4
@@ -39,50 +38,35 @@
r"(? tuple[str, str]:
+ def baseline_key(self):
return (self.path, self.fingerprint)
-@dataclasses.dataclass(frozen=True)
-class ScanResult:
- violations: tuple[Violation, ...]
- directive_errors: tuple[str, ...]
+ScanResult = collections.namedtuple("ScanResult", "violations directive_errors")
+ClassifiedViolation = collections.namedtuple(
+ "ClassifiedViolation", "violation status")
-def _consume_quoted(source: str, start: int, quote: str) -> int:
+def _consume_quoted(source, start, quote):
"""Return the first offset after a quoted Scala string or character literal."""
if quote == '"' and source.startswith('"""', start):
end = source.find('"""', start + 3)
@@ -103,7 +87,7 @@ def _consume_quoted(source: str, start: int, quote: str) -> int:
return len(source)
-def _is_interpolated_quote(source: str, quote_start: int) -> bool:
+def _is_interpolated_quote(source, quote_start):
if quote_start == 0:
return False
offset = quote_start - 1
@@ -114,7 +98,7 @@ def _is_interpolated_quote(source: str, quote_start: int) -> bool:
return source[offset].isalpha() or source[offset] in "_$"
-def _consume_block_comment(source: str, start: int) -> tuple[int, bool]:
+def _consume_block_comment(source, start):
depth = 1
offset = start + 2
while offset < len(source) and depth:
@@ -129,7 +113,7 @@ def _consume_block_comment(source: str, start: int) -> tuple[int, bool]:
return offset, depth == 0
-def _matching_interpolation_brace(source: str, open_brace: int) -> int:
+def _matching_interpolation_brace(source, open_brace):
depth = 1
offset = open_brace + 1
while offset < len(source):
@@ -156,10 +140,10 @@ def _matching_interpolation_brace(source: str, open_brace: int) -> int:
return len(source)
-def _consume_interpolated(source: str, start: int) -> tuple[int, list[tuple[int, int]]]:
+def _consume_interpolated(source, start):
delimiter = '\"\"\"' if source.startswith('\"\"\"', start) else '"'
offset = start + len(delimiter)
- expressions: list[tuple[int, int]] = []
+ expressions = []
while offset < len(source):
if source.startswith(delimiter, offset):
@@ -178,15 +162,15 @@ def _consume_interpolated(source: str, start: int) -> tuple[int, list[tuple[int,
return len(source), expressions
-def _tokenize(source: str) -> tuple[list[Token], list[LineComment]]:
+def _tokenize(source):
"""Tokenize enough Scala syntax to match calls and lexical blocks.
Comments and literal contents are deliberately opaque. This avoids counting braces or
withResource text embedded in comments and literal text. Executable `${...}` expressions
inside interpolated strings are tokenized recursively.
"""
- tokens: list[Token] = []
- line_comments: list[LineComment] = []
+ tokens = []
+ line_comments = []
offset = 0
line = 1
length = len(source)
@@ -209,9 +193,10 @@ def _tokenize(source: str) -> tuple[list[Token], list[LineComment]]:
offset, closed = _consume_block_comment(source, offset)
line += source[comment_start:offset].count("\n")
if not closed:
- raise ValueError(f"unterminated block comment at offset {comment_start}")
+ raise ValueError(
+ "unterminated block comment at offset {0}".format(comment_start))
elif char in "\"'":
- expressions: list[tuple[int, int]] = []
+ expressions = []
if char == '"' and _is_interpolated_quote(source, offset):
end, expressions = _consume_interpolated(source, offset)
else:
@@ -259,16 +244,11 @@ def _tokenize(source: str) -> tuple[list[Token], list[LineComment]]:
return tokens, line_comments
-def tokenize(source: str) -> list[Token]:
+def tokenize(source):
return _tokenize(source)[0]
-def _matching_delimiter(
- tokens: Sequence[Token],
- open_index: int,
- open_value: str,
- close_value: str,
-) -> int | None:
+def _matching_delimiter(tokens, open_index, open_value, close_value):
depth = 0
for index in range(open_index, len(tokens)):
value = tokens[index].value
@@ -281,18 +261,13 @@ def _matching_delimiter(
return None
-def _canonical_call(tokens: Sequence[Token], start: int, end: int) -> str:
+def _canonical_call(tokens, start, end):
return "".join(token.value for token in tokens[start:end + 1])
-def _directive_lines(
- source: str,
- path: str,
- tokens: Sequence[Token],
- line_comments: Sequence[LineComment],
-) -> tuple[set[int], list[str]]:
- exempt_lines: set[int] = set()
- errors: list[str] = []
+def _directive_lines(source, path, tokens, line_comments):
+ exempt_lines = set()
+ errors = []
lines = source.splitlines()
for comment in line_comments:
@@ -302,13 +277,14 @@ def _directive_lines(
line_number = comment.line
if match is None or len(match.group(1).strip()) < 10:
errors.append(
- f"{path}:{line_number}: {ALLOW_DIRECTIVE} requires a reason of at least "
- "10 characters after ' -- '")
+ "{0}:{1}: {2} requires a reason of at least "
+ "10 characters after ' -- '".format(
+ path, line_number, ALLOW_DIRECTIVE))
continue
if ISSUE_PATTERN.search(match.group(1)) is None:
errors.append(
- f"{path}:{line_number}: {ALLOW_DIRECTIVE} reason must reference an "
- "NVIDIA/cudf-spark GitHub issue by URL or #number")
+ "{0}:{1}: {2} reason must reference an NVIDIA/cudf-spark GitHub "
+ "issue by URL or #number".format(path, line_number, ALLOW_DIRECTIVE))
continue
# The directive applies to a withResource call on the same line or the next nonblank line.
@@ -326,15 +302,15 @@ def _directive_lines(
return exempt_lines, errors
-def scan_source(path: str, source: str, max_depth: int) -> ScanResult:
+def scan_source(path, source, max_depth):
try:
tokens, line_comments = _tokenize(source)
except ValueError as error:
- return ScanResult((), (f"{path}: {error}",))
+ return ScanResult((), ("{0}: {1}".format(path, error),))
exempt_lines, directive_errors = _directive_lines(
source, path, tokens, line_comments)
- resource_blocks: dict[int, ResourceCall] = {}
+ resource_blocks = {}
for index, token in enumerate(tokens):
if token.value != "withResource" or index + 1 >= len(tokens):
@@ -363,8 +339,8 @@ def scan_source(path: str, source: str, max_depth: int) -> ScanResult:
resource=canonical,
exempt=token.line in exempt_lines)
- violations: list[Violation] = []
- scope_stack: list[ResourceCall | None] = []
+ violations = []
+ scope_stack = []
for index, token in enumerate(tokens):
if token.value in {"{", "("}:
resource_call = resource_blocks.get(index)
@@ -386,77 +362,90 @@ def scan_source(path: str, source: str, max_depth: int) -> ScanResult:
return ScanResult(tuple(violations), tuple(directive_errors))
-def production_scala_files(root: Path) -> Iterable[Path]:
- for path in root.rglob("*.scala"):
- relative = path.relative_to(root)
- parts = relative.parts
- if "target" in parts or (parts and parts[0] == "scala2.13"):
- continue
- if any(parts[index:index + 2] == ("src", "main")
- for index in range(len(parts) - 1)):
- yield path
-
-
-def scan_tree(root: Path, max_depth: int) -> ScanResult:
- violations: list[Violation] = []
- directive_errors: list[str] = []
+def production_scala_files(root):
+ for directory, directory_names, file_names in os.walk(root):
+ relative_directory = os.path.relpath(directory, root)
+ parts = (() if relative_directory == "." else
+ tuple(relative_directory.split(os.sep)))
+ directory_names[:] = sorted(
+ name for name in directory_names
+ if name != "target" and not (not parts and name == "scala2.13"))
+ in_production_source = any(
+ parts[index:index + 2] == ("src", "main")
+ for index in range(len(parts) - 1))
+ if in_production_source:
+ for file_name in sorted(file_names):
+ if file_name.endswith(".scala"):
+ yield os.path.join(directory, file_name)
+
+
+def scan_tree(root, max_depth):
+ violations = []
+ directive_errors = []
for path in sorted(production_scala_files(root)):
- relative = path.relative_to(root).as_posix()
- result = scan_source(relative, path.read_text(encoding="utf-8"), max_depth)
+ relative = os.path.relpath(path, root).replace(os.sep, "/")
+ with io.open(path, "r", encoding="utf-8") as source_file:
+ result = scan_source(relative, source_file.read(), max_depth)
violations.extend(result.violations)
directive_errors.extend(result.directive_errors)
return ScanResult(tuple(violations), tuple(directive_errors))
-def load_baseline(path: Path) -> tuple[int, collections.Counter[tuple[str, str]]]:
- data = json.loads(path.read_text(encoding="utf-8"))
+def _fullmatch(pattern, value):
+ match = pattern.match(value)
+ return match is not None and match.end() == len(value)
+
+
+def load_baseline(path):
+ with io.open(path, "r", encoding="utf-8") as baseline_file:
+ data = json.loads(baseline_file.read())
if data.get("version") != BASELINE_VERSION:
raise ValueError(
- f"unsupported baseline version {data.get('version')}; expected {BASELINE_VERSION}")
+ "unsupported baseline version {0}; expected {1}".format(
+ data.get("version"), BASELINE_VERSION))
max_depth = data.get("maxDepth")
if not isinstance(max_depth, int) or max_depth < 1:
raise ValueError("baseline maxDepth must be a positive integer")
tracking_issue = data.get("trackingIssue")
- if not isinstance(tracking_issue, str) or ISSUE_PATTERN.fullmatch(tracking_issue) is None:
+ if not isinstance(tracking_issue, STRING_TYPES) or not _fullmatch(
+ ISSUE_PATTERN, tracking_issue):
raise ValueError("baseline trackingIssue must link to an NVIDIA/cudf-spark GitHub issue")
- entries: collections.Counter[tuple[str, str]] = collections.Counter()
+ entries = collections.Counter()
for entry in data.get("entries", []):
key = (entry["path"], entry["fingerprint"])
entries[key] += entry.get("count", 1)
return max_depth, entries
-def baseline_json(violations: Sequence[Violation], max_depth: int) -> str:
- grouped: dict[tuple[str, str], list[Violation]] = collections.defaultdict(list)
+def baseline_json(violations, max_depth):
+ grouped = collections.defaultdict(list)
for violation in violations:
grouped[violation.baseline_key].append(violation)
entries = []
for (path, fingerprint), matches in sorted(grouped.items()):
- entry = {
- "path": path,
- "fingerprint": fingerprint,
- "resource": matches[0].resource[:160],
- }
+ entry = collections.OrderedDict((
+ ("path", path),
+ ("fingerprint", fingerprint),
+ ("resource", matches[0].resource[:160]),
+ ))
if len(matches) > 1:
entry["count"] = len(matches)
entries.append(entry)
- return json.dumps({
- "version": BASELINE_VERSION,
- "maxDepth": max_depth,
- "trackingIssue": DEFAULT_TRACKING_ISSUE,
- "entries": entries,
- }, indent=2) + "\n"
+ baseline = collections.OrderedDict((
+ ("version", BASELINE_VERSION),
+ ("maxDepth", max_depth),
+ ("trackingIssue", DEFAULT_TRACKING_ISSUE),
+ ("entries", entries),
+ ))
+ return TEXT_TYPE(json.dumps(baseline, indent=2, separators=(",", ": "))) + "\n"
-def new_violations(
- violations: Sequence[Violation],
- baseline: collections.Counter[tuple[str, str]],
-) -> list[Violation]:
+def new_violations(violations, baseline):
remaining = baseline.copy()
- result: list[Violation] = []
+ result = []
for violation in violations:
key = violation.baseline_key
if remaining[key] > 0:
@@ -466,43 +455,149 @@ def new_violations(
return result
-def stale_baseline_entries(
- violations: Sequence[Violation],
- baseline: collections.Counter[tuple[str, str]],
-) -> collections.Counter[tuple[str, str]]:
+def stale_baseline_entries(violations, baseline):
current = collections.Counter(violation.baseline_key for violation in violations)
return baseline - current
-def parse_args(args: Sequence[str]) -> argparse.Namespace:
+def classify_violations(violations, baseline):
+ remaining = baseline.copy()
+ classified = []
+ for violation in violations:
+ key = violation.baseline_key
+ status = "baselined" if remaining[key] > 0 else "new"
+ if remaining[key] > 0:
+ remaining[key] -= 1
+ classified.append(ClassifiedViolation(violation, status))
+ return classified
+
+
+def markdown_escape(value):
+ return (value.replace("&", "&").replace("<", "<").replace(">", ">")
+ .replace("|", "\\|").replace("\r", " ").replace("\n", " "))
+
+
+def render_summary(classified, stale, directive_errors, max_depth):
+ new_count = sum(1 for item in classified if item.status == "new")
+ baselined_count = len(classified) - new_count
+ lines = [
+ "## withResource nesting audit",
+ "",
+ ("Found {0} scope(s) deeper than {1}: {2} baselined, {3} new.".format(
+ len(classified), max_depth, baselined_count, new_count)),
+ "",
+ ]
+ if classified:
+ lines.extend([
+ "| Status | Depth | Location | Resource |",
+ "| --- | ---: | --- | --- |",
+ ])
+ for item in classified:
+ violation = item.violation
+ lines.append("| {0} | {1} | `{2}:{3}` | {4} |".format(
+ item.status, violation.depth, markdown_escape(violation.path),
+ violation.line, markdown_escape(violation.resource)))
+ else:
+ lines.append("No deep withResource scopes were found.")
+
+ if stale:
+ lines.extend(["", "### Stale baseline entries", ""])
+ for (path, fingerprint), count in sorted(stale.items()):
+ lines.append("- `{0}` (`{1}`), count {2}".format(
+ markdown_escape(path), fingerprint, count))
+ if directive_errors:
+ lines.extend(["", "### Invalid exemption directives", ""])
+ lines.extend("- {0}".format(markdown_escape(error))
+ for error in directive_errors)
+ lines.extend([
+ "",
+ ("Baselined scopes are reported as existing debt and do not fail this check. "
+ "New scopes, stale baseline entries, and invalid directives fail the audit."),
+ "",
+ ])
+ return "\n".join(lines)
+
+
+def command_escape(value):
+ return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
+
+
+def command_property_escape(value):
+ return command_escape(value).replace(":", "%3A").replace(",", "%2C")
+
+
+def emit_annotations(classified):
+ for item in classified[:50]:
+ violation = item.violation
+ level = "error" if item.status == "new" else "warning"
+ message = "depth {0}: {1} ({2})".format(
+ violation.depth, violation.resource, item.status)
+ print("::{0} file={1},line={2},title=withResource nesting::{3}".format(
+ level, command_property_escape(violation.path), violation.line,
+ command_escape(message)))
+ if len(classified) > 50:
+ print("::warning title=withResource nesting::Only 50 of {0} scopes were annotated; "
+ "see the job summary and raw report for all findings".format(len(classified)))
+
+
+def write_raw_report(path, classified, stale, directive_errors, max_depth):
+ report = collections.OrderedDict((
+ ("version", BASELINE_VERSION),
+ ("maxDepth", max_depth),
+ ("violations", [collections.OrderedDict((
+ ("status", item.status),
+ ("path", item.violation.path),
+ ("line", item.violation.line),
+ ("depth", item.violation.depth),
+ ("fingerprint", item.violation.fingerprint),
+ ("resource", item.violation.resource),
+ )) for item in classified]),
+ ("staleBaselineEntries", [collections.OrderedDict((
+ ("path", path),
+ ("fingerprint", fingerprint),
+ ("count", count),
+ )) for (path, fingerprint), count in sorted(stale.items())]),
+ ("directiveErrors", list(directive_errors)),
+ ))
+ with io.open(path, "w", encoding="utf-8") as report_file:
+ report_file.write(TEXT_TYPE(json.dumps(
+ report, indent=2, separators=(",", ": "))) + "\n")
+
+
+def parse_args(args):
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--root", type=Path, default=Path.cwd(),
+ parser.add_argument("--root", default=os.getcwd(),
help="repository root (default: current directory)")
- parser.add_argument("--baseline", type=Path,
- default=Path("scripts/with_resource_nesting_baseline.json"))
+ parser.add_argument("--baseline",
+ default="scripts/with_resource_nesting_baseline.json")
parser.add_argument("--max-depth", type=int, default=None,
help="override maximum allowed depth")
parser.add_argument("--print-baseline", action="store_true",
help="print a baseline for the current source tree and exit")
parser.add_argument("--update-baseline", action="store_true",
help="replace the baseline with the current source tree")
+ parser.add_argument("--summary", default=os.environ.get("GITHUB_STEP_SUMMARY"),
+ help="write a Markdown report containing every deep scope")
+ parser.add_argument("--raw-report",
+ help="write a JSON report containing every deep scope")
return parser.parse_args(args)
-def main(argv: Sequence[str] | None = None) -> int:
+def main(argv=None):
args = parse_args(sys.argv[1:] if argv is None else argv)
- root = args.root.resolve()
+ root = os.path.abspath(args.root)
baseline_path = args.baseline
- if not baseline_path.is_absolute():
- baseline_path = root / baseline_path
+ if not os.path.isabs(baseline_path):
+ baseline_path = os.path.join(root, baseline_path)
- baseline: collections.Counter[tuple[str, str]] = collections.Counter()
+ baseline = collections.Counter()
baseline_depth = DEFAULT_MAX_DEPTH
- if baseline_path.exists():
+ if os.path.exists(baseline_path):
try:
baseline_depth, baseline = load_baseline(baseline_path)
- except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
- print(f"Invalid withResource nesting baseline: {error}", file=sys.stderr)
+ except (KeyError, TypeError, ValueError) as error:
+ print("Invalid withResource nesting baseline: {0}".format(error),
+ file=sys.stderr)
return 2
max_depth = args.max_depth if args.max_depth is not None else baseline_depth
@@ -511,41 +606,64 @@ def main(argv: Sequence[str] | None = None) -> int:
return 2
scan = scan_tree(root, max_depth)
- if scan.directive_errors:
- for error in scan.directive_errors:
- print(error, file=sys.stderr)
- return 1
generated_baseline = baseline_json(scan.violations, max_depth)
if args.print_baseline:
print(generated_baseline, end="")
return 0
if args.update_baseline:
- baseline_path.write_text(generated_baseline, encoding="utf-8")
- print(f"Updated {baseline_path} with {len(scan.violations)} violations")
+ with io.open(baseline_path, "w", encoding="utf-8") as baseline_file:
+ baseline_file.write(generated_baseline)
+ print("Updated {0} with {1} violations".format(
+ baseline_path, len(scan.violations)))
return 0
unexpected = new_violations(scan.violations, baseline)
stale = stale_baseline_entries(scan.violations, baseline)
+ classified = classify_violations(scan.violations, baseline)
+ report_failed = False
+ if args.summary:
+ try:
+ with io.open(args.summary, "w", encoding="utf-8") as summary_file:
+ summary_file.write(TEXT_TYPE(render_summary(
+ classified, stale, scan.directive_errors, max_depth)))
+ except (IOError, OSError) as error:
+ report_failed = True
+ print("Could not write withResource summary: {0}".format(error), file=sys.stderr)
+ if args.raw_report:
+ try:
+ write_raw_report(
+ args.raw_report, classified, stale, scan.directive_errors, max_depth)
+ except (IOError, OSError) as error:
+ report_failed = True
+ print("Could not write withResource raw report: {0}".format(error), file=sys.stderr)
+ if os.environ.get("GITHUB_ACTIONS") == "true":
+ emit_annotations(classified)
+
+ if scan.directive_errors:
+ for error in scan.directive_errors:
+ print(error, file=sys.stderr)
+ return 1
+
if not unexpected and not stale:
print(
- f"withResource nesting lint passed ({len(scan.violations)} baselined violations, "
- f"maximum allowed depth {max_depth})")
- return 0
+ "withResource nesting lint passed ({0} baselined violations, maximum "
+ "allowed depth {1})".format(len(scan.violations), max_depth))
+ return 1 if report_failed else 0
for violation in unexpected:
resource = violation.resource
if len(resource) > 120:
resource = resource[:117] + "..."
print(
- f"{violation.path}:{violation.line}: withResource nesting depth "
- f"{violation.depth} exceeds {max_depth}\n resource: {resource}",
+ "{0}:{1}: withResource nesting depth {2} exceeds {3}\n resource: {4}".format(
+ violation.path, violation.line, violation.depth, max_depth, resource),
file=sys.stderr)
if unexpected:
print(
- f"Found {len(unexpected)} new deep withResource scope(s). Shorten resource lifetimes "
- f"or place '// {ALLOW_DIRECTIVE} -- ' immediately before a "
- "scope whose overlap is necessary.",
+ "Found {0} new deep withResource scope(s). Shorten resource lifetimes or "
+ "place '// {1} -- ' immediately before a scope "
+ "whose overlap is necessary.".format(len(unexpected), ALLOW_DIRECTIVE),
file=sys.stderr)
if unexpected and stale:
print(
@@ -556,8 +674,8 @@ def main(argv: Sequence[str] | None = None) -> int:
if stale:
stale_count = sum(stale.values())
print(
- f"The baseline contains {stale_count} resolved violation(s). Run this check with "
- "--update-baseline to ratchet it down.",
+ "The baseline contains {0} resolved violation(s). Run this check with "
+ "--update-baseline to ratchet it down.".format(stale_count),
file=sys.stderr)
return 1
diff --git a/scripts/tests/test_check_with_resource_nesting.py b/scripts/tests/test_check_with_resource_nesting.py
index 3410c676dbc..bd5b39bfc29 100644
--- a/scripts/tests/test_check_with_resource_nesting.py
+++ b/scripts/tests/test_check_with_resource_nesting.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
# Copyright (c) 2026, NVIDIA CORPORATION.
#
@@ -14,29 +14,82 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import print_function
+
import collections
import contextlib
-import importlib.util
import io
import json
+import os
+import shutil
import sys
import tempfile
import unittest
-from pathlib import Path
-SCRIPT = Path(__file__).parents[1] / "check_with_resource_nesting.py"
-SPEC = importlib.util.spec_from_file_location("check_with_resource_nesting", SCRIPT)
-LINT = importlib.util.module_from_spec(SPEC)
-assert SPEC.loader is not None
-sys.modules[SPEC.name] = LINT
-SPEC.loader.exec_module(LINT)
+try:
+ TEXT_TYPE = unicode
+except NameError: # Python 3
+ TEXT_TYPE = str
+
+
+SCRIPT = os.path.join(os.path.dirname(os.path.dirname(__file__)),
+ "check_with_resource_nesting.py")
+try:
+ import importlib.util
+ SPEC = importlib.util.spec_from_file_location("check_with_resource_nesting", SCRIPT)
+ LINT = importlib.util.module_from_spec(SPEC)
+ sys.modules[SPEC.name] = LINT
+ SPEC.loader.exec_module(LINT)
+except ImportError: # Jython 2.7 / Python 2.7
+ import imp
+ LINT = imp.load_source("check_with_resource_nesting", SCRIPT)
+
+
+@contextlib.contextmanager
+def temporary_directory():
+ path = tempfile.mkdtemp()
+ try:
+ yield path
+ finally:
+ shutil.rmtree(path)
+
+
+class OutputSink(object):
+ def __init__(self):
+ self.parts = []
+
+ def write(self, value):
+ self.parts.append(value)
+
+ def flush(self):
+ pass
+
+ def getvalue(self):
+ return TEXT_TYPE("").join(self.parts)
+
+
+@contextlib.contextmanager
+def captured_stream(name):
+ output = OutputSink()
+ original = getattr(sys, name)
+ setattr(sys, name, output)
+ try:
+ yield output
+ finally:
+ setattr(sys, name, original)
+
+
+def write_text(path, value):
+ with io.open(path, "w", encoding="utf-8") as output_file:
+ output_file.write(TEXT_TYPE(value))
def nested_source(depth):
body = "result"
for index in reversed(range(depth)):
- body = f"withResource(make{index}()) {{ resource{index} =>\n{body}\n}}"
+ body = "withResource(make{0}()) {{ resource{0} =>\n{1}\n}}".format(
+ index, body)
return body
@@ -192,69 +245,196 @@ def test_fingerprint_does_not_depend_on_depth(self):
shallower = LINT.scan_source("Test.scala", nested_source(5), 3).violations[-1]
self.assertEqual(deep.fingerprint, shallower.fingerprint)
+ def test_baseline_json_is_stable_across_runtimes(self):
+ violation = LINT.scan_source(
+ "Test.scala", "withResource(make()) { resource => result }", 0).violations[0]
+ self.assertEqual("ba163e5cbee380205ebb", violation.fingerprint)
+ self.assertEqual("""{
+ "version": 1,
+ "maxDepth": 0,
+ "trackingIssue": "https://github.com/NVIDIA/cudf-spark/issues/11713",
+ "entries": [
+ {
+ "path": "Test.scala",
+ "fingerprint": "ba163e5cbee380205ebb",
+ "resource": "withResource(make())"
+ }
+ ]
+}
+""", LINT.baseline_json((violation,), 0))
+
+ def test_production_source_discovery_excludes_generated_and_test_trees(self):
+ with temporary_directory() as root:
+ relative_paths = (
+ "module/src/main/scala/Keep.scala",
+ "module/src/test/scala/IgnoreTest.scala",
+ "module/target/generated/src/main/scala/IgnoreTarget.scala",
+ "scala2.13/module/src/main/scala/IgnoreGeneratedPomTree.scala",
+ )
+ for relative_path in relative_paths:
+ path = os.path.join(root, *relative_path.split("/"))
+ parent = os.path.dirname(path)
+ if not os.path.isdir(parent):
+ os.makedirs(parent)
+ write_text(path, "object Fixture\n")
+
+ discovered = [
+ os.path.relpath(path, root).replace(os.sep, "/")
+ for path in LINT.production_scala_files(root)
+ ]
+ self.assertEqual(["module/src/main/scala/Keep.scala"], discovered)
+
+ def test_report_classifies_and_lists_every_violation(self):
+ violations = LINT.scan_source("Test.scala", nested_source(6), 4).violations
+ baseline = collections.Counter({violations[0].baseline_key: 1})
+ classified = LINT.classify_violations(violations, baseline)
+ self.assertEqual(["baselined", "new"], [item.status for item in classified])
+
+ summary = LINT.render_summary(classified, collections.Counter(), (), 4)
+ self.assertIn("Found 2 scope(s) deeper than 4: 1 baselined, 1 new", summary)
+ for violation in violations:
+ self.assertIn(violation.resource, summary)
+
+ with temporary_directory() as root:
+ report_path = os.path.join(root, "report.json")
+ LINT.write_raw_report(
+ report_path, classified, collections.Counter(), (), 4)
+ with io.open(report_path, "r", encoding="utf-8") as report_file:
+ report = json.loads(report_file.read())
+ self.assertEqual(2, len(report["violations"]))
+ self.assertEqual(["baselined", "new"], [
+ violation["status"] for violation in report["violations"]])
+ self.assertEqual("<literal> & value \\| next",
+ LINT.markdown_escape(" & value | next"))
+
+ def test_annotations_distinguish_baselined_and_new_violations(self):
+ violations = LINT.scan_source("Test:File.scala", nested_source(6), 4).violations
+ baseline = collections.Counter({violations[0].baseline_key: 1})
+ classified = LINT.classify_violations(violations, baseline)
+ with captured_stream("stdout") as stdout:
+ LINT.emit_annotations(classified)
+ output = stdout.getvalue()
+ self.assertIn("::warning file=Test%3AFile.scala", output)
+ self.assertIn("::error file=Test%3AFile.scala", output)
+
def test_command_fails_for_new_violation(self):
- with tempfile.TemporaryDirectory() as temp_dir:
- root = Path(temp_dir)
- source_dir = root / "module" / "src" / "main" / "scala"
- source_dir.mkdir(parents=True)
- (source_dir / "Test.scala").write_text(nested_source(5), encoding="utf-8")
- baseline = root / "baseline.json"
- baseline.write_text(json.dumps({
+ with temporary_directory() as root:
+ source_dir = os.path.join(root, "module", "src", "main", "scala")
+ os.makedirs(source_dir)
+ write_text(os.path.join(source_dir, "Test.scala"), nested_source(5))
+ baseline = os.path.join(root, "baseline.json")
+ write_text(baseline, json.dumps({
"version": 1,
"maxDepth": 4,
"trackingIssue": "https://github.com/NVIDIA/cudf-spark/issues/11713",
"entries": [],
- }), encoding="utf-8")
+ }))
- stderr = io.StringIO()
- with contextlib.redirect_stderr(stderr):
+ with captured_stream("stderr") as stderr:
exit_code = LINT.main([
- "--root", str(root),
- "--baseline", str(baseline),
+ "--root", root,
+ "--baseline", baseline,
])
self.assertEqual(1, exit_code)
self.assertIn("nesting depth 5 exceeds 4", stderr.getvalue())
def test_command_accepts_justified_exemption(self):
- with tempfile.TemporaryDirectory() as temp_dir:
- root = Path(temp_dir)
- source_dir = root / "module" / "src" / "main" / "scala"
- source_dir.mkdir(parents=True)
+ with temporary_directory() as root:
+ source_dir = os.path.join(root, "module", "src", "main", "scala")
+ os.makedirs(source_dir)
source = (
"// with-resource-lint: allow-deep-nesting -- required by "
"https://github.com/NVIDIA/cudf-spark/issues/11713\n" +
nested_source(5))
- (source_dir / "Test.scala").write_text(source, encoding="utf-8")
- baseline = root / "baseline.json"
- baseline.write_text(json.dumps({
+ write_text(os.path.join(source_dir, "Test.scala"), source)
+ baseline = os.path.join(root, "baseline.json")
+ write_text(baseline, json.dumps({
"version": 1,
"maxDepth": 4,
"trackingIssue": "https://github.com/NVIDIA/cudf-spark/issues/11713",
"entries": [],
- }), encoding="utf-8")
+ }))
- stdout = io.StringIO()
- with contextlib.redirect_stdout(stdout):
+ with captured_stream("stdout") as stdout:
exit_code = LINT.main([
- "--root", str(root),
- "--baseline", str(baseline),
+ "--root", root,
+ "--baseline", baseline,
])
self.assertEqual(0, exit_code)
self.assertIn("lint passed", stdout.getvalue())
+ def test_command_updates_baseline(self):
+ with temporary_directory() as root:
+ source_dir = os.path.join(root, "module", "src", "main", "scala")
+ os.makedirs(source_dir)
+ write_text(os.path.join(source_dir, "Test.scala"), nested_source(5))
+ baseline = os.path.join(root, "baseline.json")
+
+ with captured_stream("stdout") as stdout:
+ exit_code = LINT.main([
+ "--root", root,
+ "--baseline", baseline,
+ "--update-baseline",
+ ])
+
+ self.assertEqual(0, exit_code)
+ self.assertIn("Updated", stdout.getvalue())
+ with io.open(baseline, "r", encoding="utf-8") as baseline_file:
+ generated = baseline_file.read()
+ scan = LINT.scan_tree(root, 4)
+ self.assertEqual(LINT.baseline_json(scan.violations, 4), generated)
+
+ def test_command_writes_complete_reports(self):
+ with temporary_directory() as root:
+ source_dir = os.path.join(root, "module", "src", "main", "scala")
+ os.makedirs(source_dir)
+ write_text(os.path.join(source_dir, "Test.scala"), nested_source(6))
+ scan = LINT.scan_tree(root, 4)
+ baseline = os.path.join(root, "baseline.json")
+ write_text(baseline, LINT.baseline_json(scan.violations, 4))
+ summary = os.path.join(root, "summary.md")
+ report = os.path.join(root, "report.json")
+
+ with captured_stream("stdout"):
+ exit_code = LINT.main([
+ "--root", root,
+ "--baseline", baseline,
+ "--summary", summary,
+ "--raw-report", report,
+ ])
+
+ self.assertEqual(0, exit_code)
+ with io.open(summary, "r", encoding="utf-8") as summary_file:
+ self.assertIn("2 baselined, 0 new", summary_file.read())
+ with io.open(report, "r", encoding="utf-8") as report_file:
+ report_data = json.loads(report_file.read())
+ self.assertEqual(2, len(report_data["violations"]))
+
+ def test_command_fails_when_report_cannot_be_written(self):
+ with temporary_directory() as root:
+ baseline = os.path.join(root, "baseline.json")
+ write_text(baseline, LINT.baseline_json((), 4))
+ with captured_stream("stderr") as stderr, captured_stream("stdout"):
+ exit_code = LINT.main([
+ "--root", root,
+ "--baseline", baseline,
+ "--raw-report", root,
+ ])
+ self.assertEqual(1, exit_code)
+ self.assertIn("Could not write withResource raw report", stderr.getvalue())
+
def test_command_explains_fingerprint_changes(self):
- with tempfile.TemporaryDirectory() as temp_dir:
- root = Path(temp_dir)
- source_dir = root / "module" / "src" / "main" / "scala"
- source_dir.mkdir(parents=True)
+ with temporary_directory() as root:
+ source_dir = os.path.join(root, "module", "src", "main", "scala")
+ os.makedirs(source_dir)
source = nested_source(5)
- source_path = source_dir / "Test.scala"
- source_path.write_text(source, encoding="utf-8")
+ source_path = os.path.join(source_dir, "Test.scala")
+ write_text(source_path, source)
violation = LINT.scan_source("module/src/main/scala/Test.scala", source, 4).violations[0]
- baseline = root / "baseline.json"
- baseline.write_text(json.dumps({
+ baseline = os.path.join(root, "baseline.json")
+ write_text(baseline, json.dumps({
"version": 1,
"maxDepth": 4,
"trackingIssue": "https://github.com/NVIDIA/cudf-spark/issues/11713",
@@ -263,14 +443,13 @@ def test_command_explains_fingerprint_changes(self):
"fingerprint": violation.fingerprint,
"resource": violation.resource,
}],
- }), encoding="utf-8")
- source_path.write_text(source.replace("make4", "renamedMake4"), encoding="utf-8")
+ }))
+ write_text(source_path, source.replace("make4", "renamedMake4"))
- stderr = io.StringIO()
- with contextlib.redirect_stderr(stderr):
+ with captured_stream("stderr") as stderr:
exit_code = LINT.main([
- "--root", str(root),
- "--baseline", str(baseline),
+ "--root", root,
+ "--baseline", baseline,
])
self.assertEqual(1, exit_code)