From cad0db92adf68799663fbb627cd444670712b1bf Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Tue, 1 Sep 2026 10:27:22 -0700 Subject: [PATCH 1/5] Add advisory NVIDIA deprecation audit Signed-off-by: Gera Shegalov --- .github/workflows/mvn-verify-check.yml | 31 ++ CONTRIBUTING.md | 18 ++ pom.xml | 22 +- scala2.13/pom.xml | 22 +- scripts/check_deprecation_policy.py | 268 +++++++++++++++++ scripts/deprecation_audit.py | 373 ++++++++++++++++++++++++ scripts/tests/test_deprecation_audit.py | 195 +++++++++++++ 7 files changed, 927 insertions(+), 2 deletions(-) create mode 100644 scripts/check_deprecation_policy.py create mode 100644 scripts/deprecation_audit.py create mode 100644 scripts/tests/test_deprecation_audit.py diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 2afb1eebb7d..61370dbb7f2 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -36,6 +36,7 @@ env: -Drapids.secondaryCacheDir=$HOME/.m2/repository/.sbt/1.0/zinc/org.scala-sbt permissions: + actions: read contents: read jobs: @@ -430,3 +431,33 @@ jobs: fi } done + + nvidia-deprecation-audit: + name: NVIDIA deprecation audit + if: ${{ always() }} + continue-on-error: true + needs: + - package-tests + - package-tests-scala213 + - verify-213-modules + - verify-all-212-modules + - install-modules + runs-on: ubuntu-latest + steps: + - uses: NVIDIA/spark-rapids-common/checkout@main + + - name: Collect compiler deprecations from matrix logs + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/deprecation_audit.py \ + --repo-root "$GITHUB_WORKSPACE" \ + --raw-report "$RUNNER_TEMP/nvidia-deprecation-audit.json" + + - name: Upload deprecation report + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: nvidia-deprecation-audit + path: ${{ runner.temp }}/nvidia-deprecation-audit.json + if-no-files-found: ignore diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c52cf4ca5f2..7c5a312f250 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -235,6 +235,24 @@ or similarly ./build/buildall --rebuild-dist-only --option="-Ddist.jar.compress=false -Drapids.jni.unpack.skip" ``` +### Cross-repository API deprecations + +API replacements in cuDF Java, cudf-spark-jni, and cudf-spark-private must allow cudf-spark time to +consume a published artifact before the old entry point becomes deprecated. Introduce the +replacement first while the old method remains supported and delegates to the same implementation. +After the updated snapshot is available, migrate cudf-spark callers in a separate change. Add the +deprecation annotation only after known callers have migrated, and retain the compatibility entry +point for at least one more release before removal. + +Scala deprecations originating in NVIDIA-owned `ai.rapids.cudf`, `com.nvidia.spark.rapids`, and +`org.apache.spark.sql.rapids` APIs are reported as compiler information instead of fatal warnings +during this migration window. The same applies to cudf-spark-private's +`org.apache.spark.sql.execution.aggregate.PartialAggUtils` bridge; other APIs in Apache Spark +namespaces are not exempt. Deprecations from other dependencies remain build errors. The +non-blocking NVIDIA deprecation audit in pull requests collects these diagnostics across the Maven +build matrix; findings should result in follow-up migration work even though the audit itself does +not fail the build. + ## Code contributions ### Source code layout diff --git a/pom.xml b/pom.xml index ac0296ce52a..ce8b60ccde0 100644 --- a/pom.xml +++ b/pom.xml @@ -1743,6 +1743,18 @@ This will force full Scala code rebuild in downstream modules. -Wconf:cat=unused-privates:e -Wunused:imports,locals,patvars,privates --> + + -Wconf:cat=deprecation&origin=ai\.rapids\.cudf\..*:iv + -Wconf:cat=deprecation&origin=com\.nvidia\.spark\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.execution\.aggregate\.PartialAggUtils([.$].*|$):iv ${scala.javac.args} diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index a9d3442ef2b..b1ae3588ad9 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -1743,6 +1743,18 @@ This will force full Scala code rebuild in downstream modules. -Wconf:cat=unused-privates:e -Wunused:imports,locals,patvars,privates + + -Wconf:cat=deprecation&origin=ai\.rapids\.cudf\..*:iv + -Wconf:cat=deprecation&origin=com\.nvidia\.spark\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.execution\.aggregate\.PartialAggUtils([.$].*|$):iv ${scala.javac.args} @@ -2038,7 +2050,15 @@ This will force full Scala code rebuild in downstream modules. - + + + + + + + + diff --git a/scripts/check_deprecation_policy.py b/scripts/check_deprecation_policy.py new file mode 100644 index 00000000000..8140b98ceed --- /dev/null +++ b/scripts/check_deprecation_policy.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compile fixtures that enforce the repository's scoped deprecation policy.""" + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ElementTree +from pathlib import Path + + +MAVEN_NAMESPACE = {"m": "http://maven.apache.org/POM/4.0.0"} + + +def compiler_configuration(pom_path): + root = ElementTree.parse(pom_path).getroot() + scala_version = root.findtext("m:properties/m:scala.version", namespaces=MAVEN_NAMESPACE) + if not scala_version: + raise RuntimeError(f"Could not find scala.version in {pom_path}") + plugin_paths = ( + "m:build/m:plugins/m:plugin", + "m:build/m:pluginManagement/m:plugins/m:plugin", + ) + plugins = ( + plugin + for plugin_path in plugin_paths + for plugin in root.findall(plugin_path, MAVEN_NAMESPACE) + ) + for plugin in plugins: + artifact_id = plugin.findtext("m:artifactId", namespaces=MAVEN_NAMESPACE) + if artifact_id == "scala-maven-plugin": + args = [ + argument.text + for argument in plugin.findall("m:configuration/m:args/m:arg", MAVEN_NAMESPACE) + if argument.text + ] + if not args: + raise RuntimeError(f"scala-maven-plugin has no compiler arguments in {pom_path}") + return scala_version, args + raise RuntimeError(f"Could not find scala-maven-plugin in {pom_path}") + + +def scala_compiler_classpath(maven_repo, scala_version): + scala_root = Path(maven_repo) / "org" / "scala-lang" + jars = [ + scala_root / artifact / scala_version / f"{artifact}-{scala_version}.jar" + for artifact in ("scala-compiler", "scala-library", "scala-reflect") + ] + missing = [str(jar) for jar in jars if not jar.is_file()] + if missing: + raise RuntimeError("Missing Scala compiler dependencies: " + ", ".join(missing)) + return jars + + +def run_command(command): + return subprocess.run(command, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + + +def write_fixtures(root): + sources = { + "ai/rapids/cudf/fixture/NvidiaApi.java": """ +package ai.rapids.cudf.fixture; +public final class NvidiaApi { + private NvidiaApi() {} + @Deprecated public static void oldApi() {} +} +""", + "org/example/fixture/ThirdPartyApi.java": """ +package org.example.fixture; +public final class ThirdPartyApi { + private ThirdPartyApi() {} + @Deprecated public static void oldApi() {} +} +""", + "com/nvidia/spark/rapids/jni/fixture/JniApi.java": """ +package com.nvidia.spark.rapids.jni.fixture; +public final class JniApi { + private JniApi() {} + @Deprecated public static void oldApi() {} +} +""", + "com/nvidia/spark/rapids/optimizer/fixture/PrivateApi.java": """ +package com.nvidia.spark.rapids.optimizer.fixture; +public final class PrivateApi { + private PrivateApi() {} + @Deprecated public static void oldApi() {} +} +""", + "org/apache/spark/sql/rapids/internal/fixture/PrivateApi.java": """ +package org.apache.spark.sql.rapids.internal.fixture; +public final class PrivateApi { + private PrivateApi() {} + @Deprecated public static void oldApi() {} +} +""", + "org/apache/spark/sql/execution/aggregate/PartialAggUtils.java": """ +package org.apache.spark.sql.execution.aggregate; +public final class PartialAggUtils { + private PartialAggUtils() {} + @Deprecated public static void oldApi() {} +} +""", + "org/apache/spark/sql/execution/aggregate/PartialAggUtilsNeighbor.java": """ +package org.apache.spark.sql.execution.aggregate; +public final class PartialAggUtilsNeighbor { + private PartialAggUtilsNeighbor() {} + @Deprecated public static void oldApi() {} +} +""", + "org/apache/spark/sql/execution/aggregate/SparkApi.java": """ +package org.apache.spark.sql.execution.aggregate; +public final class SparkApi { + private SparkApi() {} + @Deprecated public static void oldApi() {} +} +""", + "NvidiaCall.scala": """ +object NvidiaCall { + def call(): Unit = ai.rapids.cudf.fixture.NvidiaApi.oldApi() +} +""", + "JniCall.scala": """ +object JniCall { + def call(): Unit = com.nvidia.spark.rapids.jni.fixture.JniApi.oldApi() +} +""", + "PrivateComNvidiaCall.scala": """ +object PrivateComNvidiaCall { + def call(): Unit = com.nvidia.spark.rapids.optimizer.fixture.PrivateApi.oldApi() +} +""", + "PrivateRapidsCall.scala": """ +object PrivateRapidsCall { + def call(): Unit = org.apache.spark.sql.rapids.internal.fixture.PrivateApi.oldApi() +} +""", + "PrivateSparkBridgeCall.scala": """ +object PrivateSparkBridgeCall { + def call(): Unit = org.apache.spark.sql.execution.aggregate.PartialAggUtils.oldApi() +} +""", + "PartialAggUtilsNeighborCall.scala": """ +object PartialAggUtilsNeighborCall { + def call(): Unit = org.apache.spark.sql.execution.aggregate.PartialAggUtilsNeighbor.oldApi() +} +""", + "SparkCall.scala": """ +object SparkCall { + def call(): Unit = org.apache.spark.sql.execution.aggregate.SparkApi.oldApi() +} +""", + "ThirdPartyCall.scala": """ +object ThirdPartyCall { + def call(): Unit = org.example.fixture.ThirdPartyApi.oldApi() +} +""", + } + for relative_path, source in sources.items(): + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source.lstrip(), encoding="utf-8") + + +def check_policy(pom_path, maven_repo): + scala_version, compiler_args = compiler_configuration(pom_path) + compiler_jar, library_jar, reflect_jar = scala_compiler_classpath( + maven_repo, scala_version) + javac = shutil.which("javac") + java = shutil.which("java") + if not javac or not java: + raise RuntimeError("Both java and javac are required for the deprecation policy check") + + with tempfile.TemporaryDirectory(prefix="cudf-spark-deprecation-policy-") as temp_dir: + fixture_root = Path(temp_dir) + classes = fixture_root / "classes" + classes.mkdir() + write_fixtures(fixture_root) + java_compile = run_command([ + javac, "-d", str(classes), + str(fixture_root / "ai/rapids/cudf/fixture/NvidiaApi.java"), + str(fixture_root / "com/nvidia/spark/rapids/jni/fixture/JniApi.java"), + str(fixture_root / "com/nvidia/spark/rapids/optimizer/fixture/PrivateApi.java"), + str(fixture_root / "org/apache/spark/sql/rapids/internal/fixture/PrivateApi.java"), + str(fixture_root / "org/apache/spark/sql/execution/aggregate/PartialAggUtils.java"), + str(fixture_root / + "org/apache/spark/sql/execution/aggregate/PartialAggUtilsNeighbor.java"), + str(fixture_root / "org/apache/spark/sql/execution/aggregate/SparkApi.java"), + str(fixture_root / "org/example/fixture/ThirdPartyApi.java"), + ]) + if java_compile.returncode: + raise RuntimeError("Could not compile Java fixtures:\n" + java_compile.stdout) + + compiler_classpath = os.pathsep.join(map(str, (compiler_jar, library_jar, reflect_jar))) + source_classpath = os.pathsep.join(map(str, (classes, library_jar))) + + def compile_scala(source): + return run_command([ + java, "-cp", compiler_classpath, "scala.tools.nsc.Main", + "-classpath", source_classpath, "-d", str(classes), + *compiler_args, str(fixture_root / source), + ]) + + nvidia_sources = ( + ("cuDF Java", "NvidiaCall.scala"), + ("cudf-spark-jni", "JniCall.scala"), + ("cudf-spark-private com.nvidia namespace", "PrivateComNvidiaCall.scala"), + ("cudf-spark-private RAPIDS namespace", "PrivateRapidsCall.scala"), + ("cudf-spark-private Spark-package bridge", "PrivateSparkBridgeCall.scala"), + ) + for api_name, source in nvidia_sources: + nvidia_compile = compile_scala(source) + if nvidia_compile.returncode or "deprecated" not in nvidia_compile.stdout.lower(): + raise RuntimeError( + f"{api_name} deprecation must be visible and nonfatal, " + "but compilation produced:\n" + nvidia_compile.stdout) + + fatal_sources = ( + ("Third-party", "ThirdPartyCall.scala"), + ("Apache Spark sibling", "SparkCall.scala"), + ("PartialAggUtils prefix neighbor", "PartialAggUtilsNeighborCall.scala"), + ) + for api_name, source in fatal_sources: + fatal_compile = compile_scala(source) + if fatal_compile.returncode == 0 or "deprecated" not in fatal_compile.stdout.lower(): + raise RuntimeError( + f"{api_name} deprecation must be visible and fatal, " + "but compilation produced:\n" + fatal_compile.stdout) + + print(f"Deprecation policy check passed for Scala {scala_version}") + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pom", required=True) + parser.add_argument("--maven-repo", required=True) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + try: + check_policy(args.pom, args.maven_repo) + except RuntimeError as error: + print(error, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/deprecation_audit.py b/scripts/deprecation_audit.py new file mode 100644 index 00000000000..ce123ed347d --- /dev/null +++ b/scripts/deprecation_audit.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collect compiler deprecation diagnostics from a GitHub Actions build matrix.""" + +import argparse +import io +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from dataclasses import dataclass, field +from pathlib import Path + + +ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") +SOURCE_LOCATION = re.compile( + r"(?P(?:[A-Za-z]:)?[^\s\[\]]+?\.(?:scala|java))" + r"(?::(?P\d+)|:\[(?P\d+),\d+\])" +) +DEPRECATION = re.compile(r"\bdeprecated\b", re.IGNORECASE) +ORIGIN = re.compile(r"\borigin(?:=|:)\s*(?P[\w.$]+)") +DEFAULT_JOB_PATTERN = ( + r"^(?:package-tests(?:-scala213)?|verify-213-modules|" + r"verify-all-212-modules|install-modules)(?:\s|$)" +) +NVIDIA_ORIGIN_PREFIXES = ( + "ai.rapids.cudf.", + "com.nvidia.spark.rapids.", + "org.apache.spark.sql.rapids.", +) +NVIDIA_ORIGIN_SYMBOLS = ( + "org.apache.spark.sql.execution.aggregate.PartialAggUtils", +) + + +def is_nvidia_origin(origin): + if origin.startswith(NVIDIA_ORIGIN_PREFIXES): + return True + return any( + origin == symbol or origin.startswith((symbol + ".", symbol + "$")) + for symbol in NVIDIA_ORIGIN_SYMBOLS + ) + + +@dataclass +class Finding: + path: str + line: int + message: str + origin: str = "" + jobs: set[str] = field(default_factory=set) + + @property + def owner(self): + if is_nvidia_origin(self.origin): + return "NVIDIA" + return "third-party/unknown" + + def key(self): + diagnostic = self.origin or self.message + return self.path, self.line, diagnostic + + +def clean_line(line): + return ANSI_ESCAPE.sub("", line).rstrip() + + +def normalize_path(path, repo_root): + candidate = Path(path) + if not candidate.is_absolute(): + return candidate.as_posix() + root = Path(repo_root).resolve() + try: + return candidate.resolve().relative_to(root).as_posix() + except ValueError: + pass + parts = candidate.parts + for index in range(len(parts)): + suffix = Path(*parts[index:]) + if (root / suffix).exists(): + return suffix.as_posix() + return candidate.as_posix() + + +def parse_log(text, job_name, repo_root="."): + lines = [clean_line(line) for line in text.splitlines()] + findings = [] + for index, line in enumerate(lines): + if not DEPRECATION.search(line): + continue + location = SOURCE_LOCATION.search(line) + if location is None: + for previous in reversed(lines[max(0, index - 3):index]): + location = SOURCE_LOCATION.search(previous) + if location is not None: + break + if location is None: + continue + origin = "" + for context in lines[index:min(len(lines), index + 6)]: + origin_match = ORIGIN.search(context) + if origin_match is not None: + origin = origin_match.group("origin") + break + line_number = location.group("line") or location.group("bracket_line") + message = re.sub(r"^.*?\.(?:scala|java)(?::\d+|:\[\d+,\d+\])\s*:?[ ]*", "", line) + findings.append(Finding( + path=normalize_path(location.group("path"), repo_root), + line=int(line_number), + message=message.strip() or line.strip(), + origin=origin, + jobs={job_name}, + )) + return findings + + +def merge_findings(findings): + merged = {} + for finding in findings: + existing = merged.get(finding.key()) + if existing is None: + merged[finding.key()] = finding + else: + existing.jobs.update(finding.jobs) + return sorted(merged.values(), key=lambda finding: (finding.path, finding.line, finding.message)) + + +def request_json(url, token): + request = urllib.request.Request(url, headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +class JobLogRedirectError(RuntimeError): + """The GitHub job-log endpoint returned an unsafe or malformed redirect.""" + + +class NoRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +def request_bytes(url, token): + """Download a GitHub API resource without forwarding credentials on its redirect.""" + request = urllib.request.Request(url, headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }) + opener = urllib.request.build_opener(NoRedirectHandler()) + try: + with opener.open(request, timeout=30): + raise JobLogRedirectError( + "GitHub job-log endpoint did not return the expected redirect") + except urllib.error.HTTPError as error: + if error.code != 302: + error.close() + raise + location = error.headers.get("Location") + error.close() + if not location: + raise JobLogRedirectError( + "GitHub job-log redirect did not include a Location header") + parsed_location = urllib.parse.urlsplit(location) + if parsed_location.scheme != "https" or not parsed_location.netloc: + raise JobLogRedirectError( + "GitHub job-log redirect must use an absolute HTTPS URL") + + # The redirect is a short-lived signed URL. It authorizes itself, so use a fresh request + # without the repository-scoped GitHub token or GitHub-specific API headers. + signed_request = urllib.request.Request(location) + with urllib.request.urlopen(signed_request, timeout=30) as response: + return response.read() +def decode_job_log(payload): + if payload.startswith(b"PK"): + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + return "\n".join( + archive.read(name).decode("utf-8", errors="replace") + for name in archive.namelist() + if not name.endswith("/") + ) + return payload.decode("utf-8", errors="replace") + + +def download_logs(api_url, repository, run_id, token, job_pattern): + matcher = re.compile(job_pattern) + jobs = [] + page = 1 + while True: + result = request_json( + f"{api_url}/repos/{repository}/actions/runs/{run_id}/jobs?per_page=100&page={page}", + token, + ) + page_jobs = result.get("jobs", []) + jobs.extend(page_jobs) + if len(page_jobs) < 100: + break + page += 1 + + logs = {} + failures = [] + for job in jobs: + name = job.get("name", "") + if job.get("status") != "completed" or matcher.search(name) is None: + continue + error = None + for delay in (0, 1, 2, 4): + if delay: + time.sleep(delay) + try: + payload = request_bytes( + f"{api_url}/repos/{repository}/actions/jobs/{job['id']}/logs", token) + logs[name] = decode_job_log(payload) + error = None + break + except (OSError, urllib.error.HTTPError, zipfile.BadZipFile, + JobLogRedirectError) as caught: + error = caught + if error is not None: + failures.append(f"{name}: {error}") + if not logs and not failures: + failures.append("no completed build-matrix job logs matched the configured job pattern") + return logs, failures + + +def markdown_escape(value): + return value.replace("|", "\\|").replace("\n", " ") + + +def render_summary(findings, failures): + lines = ["## NVIDIA deprecation audit", ""] + if findings: + lines.extend([ + f"Found {len(findings)} unique compiler deprecation diagnostic(s).", + "", + "| Owner | Location | Deprecated API | Matrix jobs |", + "| --- | --- | --- | --- |", + ]) + for finding in findings[:200]: + location = f"`{finding.path}:{finding.line}`" + api = finding.origin or finding.message + jobs = ", ".join(sorted(finding.jobs)) + lines.append( + f"| {finding.owner} | {location} | `{markdown_escape(api)}` | " + f"{markdown_escape(jobs)} |" + ) + if len(findings) > 200: + lines.extend(["", f"Report truncated; see the raw artifact for all {len(findings)} findings."]) + else: + lines.append("No compiler deprecation diagnostics were found in the selected matrix jobs.") + if failures: + lines.extend(["", "### Incomplete log collection", ""]) + lines.extend(f"- {markdown_escape(failure)}" for failure in failures) + lines.extend([ + "", + "This audit is advisory. Build-job failures remain authoritative.", + "", + ]) + return "\n".join(lines) + + +def command_escape(value): + return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def command_property_escape(value): + return command_escape(value).replace(":", "%3A").replace(",", "%2C") + + +def emit_annotations(findings): + for finding in findings[:50]: + message = finding.origin or finding.message + print( + f"::warning file={command_property_escape(finding.path)},line={finding.line}," + f"title=NVIDIA deprecation::{command_escape(message)}" + ) + if len(findings) > 50: + print(f"::warning title=NVIDIA deprecation::Only 50 of {len(findings)} findings were annotated") + + +def write_raw_report(path, findings, failures): + report = { + "findings": [ + { + "owner": finding.owner, + "path": finding.path, + "line": finding.line, + "message": finding.message, + "origin": finding.origin, + "jobs": sorted(finding.jobs), + } + for finding in findings + ], + "log_collection_failures": failures, + } + Path(path).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY")) + parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID")) + parser.add_argument("--api-url", default=os.environ.get("GITHUB_API_URL", "https://api.github.com")) + parser.add_argument("--job-pattern", default=DEFAULT_JOB_PATTERN) + parser.add_argument("--logs-dir", help="Parse local *.log files instead of downloading job logs") + parser.add_argument("--repo-root", default=".") + parser.add_argument("--summary", default=os.environ.get("GITHUB_STEP_SUMMARY")) + parser.add_argument("--raw-report", default="nvidia-deprecation-audit.json") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + failures = [] + try: + if args.logs_dir: + logs = { + path.stem: path.read_text(encoding="utf-8", errors="replace") + for path in Path(args.logs_dir).glob("*.log") + } + else: + token = os.environ.get("GITHUB_TOKEN") + if not token or not args.repository or not args.run_id: + raise ValueError("GITHUB_TOKEN, repository, and run ID are required") + logs, failures = download_logs( + args.api_url, args.repository, args.run_id, token, args.job_pattern) + findings = merge_findings( + finding + for job_name, log in logs.items() + for finding in parse_log(log, job_name, args.repo_root) + ) + except Exception as error: # The audit must never mask the build result. + findings = [] + failures.append(f"audit failed: {error}") + + summary = render_summary(findings, failures) + print(summary) + emit_annotations(findings) + if args.summary: + with Path(args.summary).open("a", encoding="utf-8") as summary_file: + summary_file.write(summary) + try: + write_raw_report(args.raw_report, findings, failures) + except OSError as error: + print(f"::warning title=NVIDIA deprecation audit::Could not write raw report: {error}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_deprecation_audit.py b/scripts/tests/test_deprecation_audit.py new file mode 100644 index 00000000000..9a808c81f1b --- /dev/null +++ b/scripts/tests/test_deprecation_audit.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import io +import sys +import unittest +import zipfile +from pathlib import Path +from unittest import mock + + +SCRIPT = Path(__file__).parents[1] / "deprecation_audit.py" +SPEC = importlib.util.spec_from_file_location("deprecation_audit", SCRIPT) +AUDIT = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = AUDIT +SPEC.loader.exec_module(AUDIT) + + +class DeprecationAuditSuite(unittest.TestCase): + def test_parses_scala_verbose_deprecation(self): + log = ( + "[INFO] /workspace/sql-plugin/src/main/scala/Test.scala:42: " + "[deprecation @ example.Test.run | " + "origin=ai.rapids.cudf.ColumnView.oldApi | version=] " + "method oldApi in class ColumnView is deprecated\n" + ) + findings = AUDIT.parse_log(log, "package-tests (330)") + self.assertEqual(1, len(findings)) + self.assertEqual(42, findings[0].line) + self.assertEqual("ai.rapids.cudf.ColumnView.oldApi", findings[0].origin) + self.assertEqual("NVIDIA", findings[0].owner) + + def test_parses_maven_bracket_location(self): + log = """ +[WARNING] /workspace/src/main/java/Test.java:[17,9] oldApi() has been deprecated +""" + findings = AUDIT.parse_log(log, "verify-all-212-modules (330, 17)") + self.assertEqual(17, findings[0].line) + self.assertEqual("third-party/unknown", findings[0].owner) + + def test_reads_scala_213_origin_from_following_line(self): + log = """ +[INFO] /workspace/sql-plugin/src/main/scala/Test.scala:42: method oldApi is deprecated +Applicable -Wconf filters: cat=deprecation, origin=com.nvidia.spark.rapids.jni.Api.oldApi +""" + findings = AUDIT.parse_log(log, "package-tests-scala213 (350)") + self.assertEqual("com.nvidia.spark.rapids.jni.Api.oldApi", findings[0].origin) + self.assertEqual("NVIDIA", findings[0].owner) + + def test_classifies_all_advisory_origins_as_nvidia(self): + origins = ( + "ai.rapids.cudf.ColumnView.oldApi", + "com.nvidia.spark.rapids.optimizer.OptimizerConf.oldApi", + "org.apache.spark.sql.rapids.internal.PrivateRapidsConfs.oldApi", + "org.apache.spark.sql.execution.aggregate.PartialAggUtils.oldApi", + "org.apache.spark.sql.execution.aggregate.PartialAggUtils$Helper.oldApi", + ) + for origin in origins: + with self.subTest(origin=origin): + finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) + self.assertEqual("NVIDIA", finding.owner) + + def test_partial_agg_utils_owner_match_has_symbol_boundary(self): + origins = ( + "org.apache.spark.sql.execution.aggregate.PartialAggUtilsNeighbor.oldApi", + "org.apache.spark.sql.execution.aggregate.SparkApi.oldApi", + "org.example.fixture.ThirdPartyApi.oldApi", + ) + for origin in origins: + with self.subTest(origin=origin): + finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) + self.assertEqual("third-party/unknown", finding.owner) + + def test_ignores_non_source_deprecation_text(self): + log = "[WARNING] This build plugin uses a deprecated Maven feature\n" + self.assertEqual([], AUDIT.parse_log(log, "install-modules (3.9.3)")) + + def test_merges_same_finding_across_matrix_jobs(self): + first = AUDIT.Finding( + "Test.scala", 1, "[deprecation] old is deprecated", "ai.rapids.cudf.Api.old", + {"330"}) + second = AUDIT.Finding( + "Test.scala", 1, "old is deprecated", "ai.rapids.cudf.Api.old", {"400"}) + merged = AUDIT.merge_findings([first, second]) + self.assertEqual({"330", "400"}, merged[0].jobs) + + def test_decodes_zip_job_log(self): + payload = io.BytesIO() + with zipfile.ZipFile(payload, "w") as archive: + archive.writestr("job/step.txt", "deprecated output") + self.assertEqual("deprecated output", AUDIT.decode_job_log(payload.getvalue())) + + def test_job_log_redirect_does_not_forward_authorization(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + signed_url = "https://results-receiver.example/job.log?signature=secret" + redirect = AUDIT.urllib.error.HTTPError( + api_url, 302, "Found", {"Location": signed_url}, None) + authenticated_opener = mock.Mock() + authenticated_opener.open.side_effect = redirect + + with mock.patch.object( + AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ + mock.patch.object( + AUDIT.urllib.request, "urlopen", return_value=io.BytesIO(b"job log")) \ + as signed_open: + payload = AUDIT.request_bytes(api_url, "github-token") + + self.assertEqual(b"job log", payload) + authenticated_request = authenticated_opener.open.call_args.args[0] + self.assertEqual("Bearer github-token", + authenticated_request.get_header("Authorization")) + signed_request = signed_open.call_args.args[0] + self.assertEqual(signed_url, signed_request.full_url) + self.assertIsNone(signed_request.get_header("Authorization")) + self.assertIsNone(signed_request.get_header("X-GitHub-Api-Version")) + + def test_job_log_redirect_requires_location(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + authenticated_opener = mock.Mock() + authenticated_opener.open.side_effect = AUDIT.urllib.error.HTTPError( + api_url, 302, "Found", {}, None) + with mock.patch.object( + AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ + self.assertRaisesRegex(AUDIT.JobLogRedirectError, "Location"): + AUDIT.request_bytes(api_url, "github-token") + + def test_job_log_redirect_rejects_non_https_location(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + authenticated_opener = mock.Mock() + authenticated_opener.open.side_effect = AUDIT.urllib.error.HTTPError( + api_url, 302, "Found", {"Location": "http://logs.example/job.log"}, None) + with mock.patch.object( + AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ + self.assertRaisesRegex(AUDIT.JobLogRedirectError, "HTTPS"): + AUDIT.request_bytes(api_url, "github-token") + + def test_job_log_endpoint_rejects_non_redirect_response(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + authenticated_opener = mock.Mock() + authenticated_opener.open.return_value = io.BytesIO(b"unexpected direct response") + with mock.patch.object( + AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ + self.assertRaisesRegex(AUDIT.JobLogRedirectError, "expected redirect"): + AUDIT.request_bytes(api_url, "github-token") + + def test_download_logs_preserves_partial_results_after_bad_redirect(self): + jobs = {"jobs": [ + {"id": 1, "name": "package-tests (330, false)", "status": "completed"}, + {"id": 2, "name": "package-tests (340, false)", "status": "completed"}, + ]} + bad_redirect = AUDIT.JobLogRedirectError("missing redirect location") + with mock.patch.object(AUDIT, "request_json", return_value=jobs), \ + mock.patch.object( + AUDIT, "request_bytes", + side_effect=[b"first job log", bad_redirect, bad_redirect, + bad_redirect, bad_redirect]) as request_bytes, \ + mock.patch.object(AUDIT.time, "sleep"): + logs, failures = AUDIT.download_logs( + "https://api.github.com", "NVIDIA/cudf-spark", "run-id", "token", + AUDIT.DEFAULT_JOB_PATTERN) + + self.assertEqual({"package-tests (330, false)": "first job log"}, logs) + self.assertEqual(5, request_bytes.call_count) + self.assertEqual(1, len(failures)) + self.assertIn("package-tests (340, false)", failures[0]) + self.assertIn("missing redirect location", failures[0]) + + def test_summary_reports_incomplete_collection(self): + summary = AUDIT.render_summary([], ["package-tests: log unavailable"]) + self.assertIn("No compiler deprecation diagnostics", summary) + self.assertIn("Incomplete log collection", summary) + self.assertIn("advisory", summary) + + def test_annotation_property_escaping(self): + self.assertEqual("path%3Awith%2Cpunctuation", AUDIT.command_property_escape( + "path:with,punctuation")) + + +if __name__ == "__main__": + unittest.main() From 4c28634a2b378cf9a629b996fd7500e261355cf2 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Tue, 1 Sep 2026 11:20:37 -0700 Subject: [PATCH 2/5] Fail optional audit when deprecations are found Signed-off-by: Gera Shegalov --- .github/workflows/mvn-verify-check.yml | 3 +- CONTRIBUTING.md | 6 ++-- scripts/deprecation_audit.py | 9 ++++-- scripts/tests/test_deprecation_audit.py | 41 ++++++++++++++++++++++++- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 61370dbb7f2..b5f9ce0afad 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -433,9 +433,8 @@ jobs: done nvidia-deprecation-audit: - name: NVIDIA deprecation audit + name: NVIDIA deprecation audit (optional) if: ${{ always() }} - continue-on-error: true needs: - package-tests - package-tests-scala213 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c5a312f250..e71e4111ee2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -249,9 +249,9 @@ Scala deprecations originating in NVIDIA-owned `ai.rapids.cudf`, `com.nvidia.spa during this migration window. The same applies to cudf-spark-private's `org.apache.spark.sql.execution.aggregate.PartialAggUtils` bridge; other APIs in Apache Spark namespaces are not exempt. Deprecations from other dependencies remain build errors. The -non-blocking NVIDIA deprecation audit in pull requests collects these diagnostics across the Maven -build matrix; findings should result in follow-up migration work even though the audit itself does -not fail the build. +optional NVIDIA deprecation audit in pull requests collects these diagnostics across the Maven +build matrix. Findings or incomplete log collection fail the audit check so contributors inspect +the result, but the audit is not a required build check; build-job results remain authoritative. ## Code contributions diff --git a/scripts/deprecation_audit.py b/scripts/deprecation_audit.py index ce123ed347d..72906708500 100644 --- a/scripts/deprecation_audit.py +++ b/scripts/deprecation_audit.py @@ -276,7 +276,8 @@ def render_summary(findings, failures): lines.extend(f"- {markdown_escape(failure)}" for failure in failures) lines.extend([ "", - "This audit is advisory. Build-job failures remain authoritative.", + "This check is optional. Findings or incomplete log collection fail only this audit " + "check; build-job results remain authoritative.", "", ]) return "\n".join(lines) @@ -352,7 +353,7 @@ def main(argv=None): for job_name, log in logs.items() for finding in parse_log(log, job_name, args.repo_root) ) - except Exception as error: # The audit must never mask the build result. + except Exception as error: # Report operational errors through this optional check. findings = [] failures.append(f"audit failed: {error}") @@ -362,11 +363,13 @@ def main(argv=None): if args.summary: with Path(args.summary).open("a", encoding="utf-8") as summary_file: summary_file.write(summary) + report_failed = False try: write_raw_report(args.raw_report, findings, failures) except OSError as error: + report_failed = True print(f"::warning title=NVIDIA deprecation audit::Could not write raw report: {error}") - return 0 + return 1 if findings or failures or report_failed else 0 if __name__ == "__main__": diff --git a/scripts/tests/test_deprecation_audit.py b/scripts/tests/test_deprecation_audit.py index 9a808c81f1b..ff4776cb943 100644 --- a/scripts/tests/test_deprecation_audit.py +++ b/scripts/tests/test_deprecation_audit.py @@ -17,6 +17,7 @@ import importlib.util import io import sys +import tempfile import unittest import zipfile from pathlib import Path @@ -184,7 +185,45 @@ def test_summary_reports_incomplete_collection(self): summary = AUDIT.render_summary([], ["package-tests: log unavailable"]) self.assertIn("No compiler deprecation diagnostics", summary) self.assertIn("Incomplete log collection", summary) - self.assertIn("advisory", summary) + self.assertIn("optional", summary) + + def test_main_fails_when_findings_are_present(self): + log = ( + "/workspace/sql-plugin/src/main/scala/Test.scala:42: " + "[deprecation @ example.Test.run | " + "origin=ai.rapids.cudf.ColumnView.oldApi | version=] deprecated\n" + ) + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "package-tests.log").write_text(log, encoding="utf-8") + result = AUDIT.main([ + "--logs-dir", str(root), + "--raw-report", str(root / "report.json"), + ]) + self.assertEqual(1, result) + + def test_main_succeeds_when_audit_is_clean(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "package-tests.log").write_text("clean build\n", encoding="utf-8") + result = AUDIT.main([ + "--logs-dir", str(root), + "--raw-report", str(root / "report.json"), + ]) + self.assertEqual(0, result) + + def test_main_fails_when_log_collection_is_incomplete(self): + with tempfile.TemporaryDirectory() as temp_dir, \ + mock.patch.object( + AUDIT, "download_logs", + return_value=({}, ["package-tests: log unavailable"])), \ + mock.patch.dict(AUDIT.os.environ, {"GITHUB_TOKEN": "token"}): + result = AUDIT.main([ + "--repository", "NVIDIA/cudf-spark", + "--run-id", "123", + "--raw-report", str(Path(temp_dir) / "report.json"), + ]) + self.assertEqual(1, result) def test_annotation_property_escaping(self): self.assertEqual("path%3Awith%2Cpunctuation", AUDIT.command_property_escape( From 910bd0ddab830af9f3977961e5b15b70de352792 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Tue, 1 Sep 2026 11:46:37 -0700 Subject: [PATCH 3/5] Run deprecation checks with Jython Signed-off-by: Gera Shegalov --- .github/workflows/mvn-verify-check.yml | 5 +- pom.xml | 35 +++- scala2.13/pom.xml | 35 +++- scripts/check_deprecation_policy.py | 142 +++++++++----- scripts/deprecation_audit.py | 192 +++++++++++-------- scripts/tests/test_deprecation_audit.py | 242 +++++++++++++++++------- 6 files changed, 447 insertions(+), 204 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index b5f9ce0afad..54c3d2b54e2 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -449,9 +449,8 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - python3 scripts/deprecation_audit.py \ - --repo-root "$GITHUB_WORKSPACE" \ - --raw-report "$RUNNER_TEMP/nvidia-deprecation-audit.json" + mvn --batch-mode -N antrun:run@nvidia-deprecation-audit \ + -Ddeprecation.audit.rawReport="$RUNNER_TEMP/nvidia-deprecation-audit.json" - name: Upload deprecation report if: ${{ always() }} diff --git a/pom.xml b/pom.xml index ce8b60ccde0..ff8be39f99d 100644 --- a/pom.xml +++ b/pom.xml @@ -1153,6 +1153,8 @@ 4.9.10 3.1.1 3.3.0 + 2.7.3 + ${project.build.directory}/nvidia-deprecation-audit.json 2.0.2 30.0-jre 2.0.0 @@ -1619,7 +1621,7 @@ This will force full Scala code rebuild in downstream modules. org.python jython-standalone - 2.7.3 + ${jython.version} net.sourceforge.pmd @@ -2005,6 +2007,29 @@ This will force full Scala code rebuild in downstream modules. false + + nvidia-deprecation-audit + none + run + + + + + + + + + + + + + + + + + 4.9.10 3.1.1 3.3.0 + 2.7.3 + ${project.build.directory}/nvidia-deprecation-audit.json 2.0.2 30.0-jre 2.0.0 @@ -1619,7 +1621,7 @@ This will force full Scala code rebuild in downstream modules. org.python jython-standalone - 2.7.3 + ${jython.version} net.sourceforge.pmd @@ -2005,6 +2007,29 @@ This will force full Scala code rebuild in downstream modules. false + + nvidia-deprecation-audit + none + run + + + + + + + + + + + + + + + + + 200: - lines.extend(["", f"Report truncated; see the raw artifact for all {len(findings)} findings."]) + lines.extend([ + u"", + u"Report truncated; see the raw artifact for all {0} findings.".format( + len(findings)), + ]) else: - lines.append("No compiler deprecation diagnostics were found in the selected matrix jobs.") + lines.append(u"No compiler deprecation diagnostics were found in the selected matrix jobs.") if failures: - lines.extend(["", "### Incomplete log collection", ""]) - lines.extend(f"- {markdown_escape(failure)}" for failure in failures) + lines.extend([u"", u"### Incomplete log collection", u""]) + lines.extend(u"- {0}".format(markdown_escape(failure)) for failure in failures) lines.extend([ - "", - "This check is optional. Findings or incomplete log collection fail only this audit " + u"", + u"This check is optional. Findings or incomplete log collection fail only this audit " "check; build-job results remain authoritative.", - "", + u"", ]) - return "\n".join(lines) + return u"\n".join(lines) def command_escape(value): @@ -295,11 +329,12 @@ def emit_annotations(findings): for finding in findings[:50]: message = finding.origin or finding.message print( - f"::warning file={command_property_escape(finding.path)},line={finding.line}," - f"title=NVIDIA deprecation::{command_escape(message)}" + u"::warning file={0},line={1},title=NVIDIA deprecation::{2}".format( + command_property_escape(finding.path), finding.line, command_escape(message)) ) if len(findings) > 50: - print(f"::warning title=NVIDIA deprecation::Only 50 of {len(findings)} findings were annotated") + print(u"::warning title=NVIDIA deprecation::Only 50 of {0} findings were annotated".format( + len(findings))) def write_raw_report(path, findings, failures): @@ -317,16 +352,20 @@ def write_raw_report(path, findings, failures): ], "log_collection_failures": failures, } - Path(path).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + with io.open(path, "w", encoding="utf-8") as report_file: + report_file.write(json.dumps(report, indent=2, ensure_ascii=False) + u"\n") def parse_args(argv): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY")) parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID")) - parser.add_argument("--api-url", default=os.environ.get("GITHUB_API_URL", "https://api.github.com")) + parser.add_argument( + "--api-url", default=os.environ.get("GITHUB_API_URL", "https://api.github.com")) parser.add_argument("--job-pattern", default=DEFAULT_JOB_PATTERN) - parser.add_argument("--logs-dir", help="Parse local *.log files instead of downloading job logs") + parser.add_argument( + "--logs-dir", default=os.environ.get("DEPRECATION_AUDIT_LOGS_DIR"), + help="Parse local *.log files instead of downloading job logs") parser.add_argument("--repo-root", default=".") parser.add_argument("--summary", default=os.environ.get("GITHUB_STEP_SUMMARY")) parser.add_argument("--raw-report", default="nvidia-deprecation-audit.json") @@ -339,8 +378,8 @@ def main(argv=None): try: if args.logs_dir: logs = { - path.stem: path.read_text(encoding="utf-8", errors="replace") - for path in Path(args.logs_dir).glob("*.log") + os.path.splitext(os.path.basename(path))[0]: read_text(path) + for path in glob.glob(os.path.join(args.logs_dir, "*.log")) } else: token = os.environ.get("GITHUB_TOKEN") @@ -355,20 +394,21 @@ def main(argv=None): ) except Exception as error: # Report operational errors through this optional check. findings = [] - failures.append(f"audit failed: {error}") + failures.append(u"audit failed: {0}".format(error)) summary = render_summary(findings, failures) print(summary) emit_annotations(findings) if args.summary: - with Path(args.summary).open("a", encoding="utf-8") as summary_file: + with io.open(args.summary, "a", encoding="utf-8") as summary_file: summary_file.write(summary) report_failed = False try: write_raw_report(args.raw_report, findings, failures) - except OSError as error: + except (IOError, OSError) as error: report_failed = True - print(f"::warning title=NVIDIA deprecation audit::Could not write raw report: {error}") + print(u"::warning title=NVIDIA deprecation audit::Could not write raw report: {0}".format( + error)) return 1 if findings or failures or report_failed else 0 diff --git a/scripts/tests/test_deprecation_audit.py b/scripts/tests/test_deprecation_audit.py index ff4776cb943..b6e20f820a0 100644 --- a/scripts/tests/test_deprecation_audit.py +++ b/scripts/tests/test_deprecation_audit.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright (c) 2026, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,22 +12,114 @@ # See the License for the specific language governing permissions and # limitations under the License. -import importlib.util +from __future__ import print_function + +import contextlib import io +import os +import shutil import sys import tempfile import unittest import zipfile -from pathlib import Path -from unittest import mock -SCRIPT = Path(__file__).parents[1] / "deprecation_audit.py" -SPEC = importlib.util.spec_from_file_location("deprecation_audit", SCRIPT) -AUDIT = importlib.util.module_from_spec(SPEC) -assert SPEC.loader is not None -sys.modules[SPEC.name] = AUDIT -SPEC.loader.exec_module(AUDIT) +SCRIPT = os.path.join(os.path.dirname(os.path.dirname(__file__)), "deprecation_audit.py") +try: + import importlib.util + SPEC = importlib.util.spec_from_file_location("deprecation_audit", SCRIPT) + AUDIT = importlib.util.module_from_spec(SPEC) + sys.modules[SPEC.name] = AUDIT + SPEC.loader.exec_module(AUDIT) +except ImportError: # Jython 2.7 / Python 2.7 + import imp + AUDIT = imp.load_source("deprecation_audit", SCRIPT) + + +class CallRecorder(object): + def __init__(self, return_value=None, side_effect=None): + self.return_value = return_value + self.side_effect = side_effect + self.calls = [] + + @property + def call_count(self): + return len(self.calls) + + def __call__(self, *args, **kwargs): + self.calls.append((args, kwargs)) + effect = self.side_effect + if isinstance(effect, list): + effect = effect.pop(0) + if isinstance(effect, BaseException): + raise effect + if callable(effect): + return effect(*args, **kwargs) + return self.return_value if effect is None else effect + + +class FakeOpener(object): + def __init__(self): + self.open = CallRecorder() + + +class OutputSink(object): + def __init__(self): + self.parts = [] + + def write(self, value): + self.parts.append(value) + + def flush(self): + pass + + +@contextlib.contextmanager +def patch_attribute(target, name, value): + original = getattr(target, name) + setattr(target, name, value) + try: + yield value + finally: + setattr(target, name, original) + + +@contextlib.contextmanager +def patch_environment(updates): + original = dict(os.environ) + os.environ.update(updates) + try: + yield + finally: + os.environ.clear() + os.environ.update(original) + + +@contextlib.contextmanager +def temporary_directory(): + path = tempfile.mkdtemp() + try: + yield path + finally: + shutil.rmtree(path) + + +@contextlib.contextmanager +def captured_stdout(): + output = OutputSink() + with patch_attribute(sys, "stdout", output): + yield output + + +def request_url(request): + return request.get_full_url() + + +def assert_raises_regex(test_case, exception, pattern): + method = getattr(test_case, "assertRaisesRegex", None) + if method is None: + method = test_case.assertRaisesRegexp + return method(exception, pattern) class DeprecationAuditSuite(unittest.TestCase): @@ -72,9 +162,8 @@ def test_classifies_all_advisory_origins_as_nvidia(self): "org.apache.spark.sql.execution.aggregate.PartialAggUtils$Helper.oldApi", ) for origin in origins: - with self.subTest(origin=origin): - finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) - self.assertEqual("NVIDIA", finding.owner) + finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) + self.assertEqual("NVIDIA", finding.owner, origin) def test_partial_agg_utils_owner_match_has_symbol_boundary(self): origins = ( @@ -83,9 +172,8 @@ def test_partial_agg_utils_owner_match_has_symbol_boundary(self): "org.example.fixture.ThirdPartyApi.oldApi", ) for origin in origins: - with self.subTest(origin=origin): - finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) - self.assertEqual("third-party/unknown", finding.owner) + finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) + self.assertEqual("third-party/unknown", finding.owner, origin) def test_ignores_non_source_deprecation_text(self): log = "[WARNING] This build plugin uses a deprecated Maven feature\n" @@ -109,54 +197,56 @@ def test_decodes_zip_job_log(self): def test_job_log_redirect_does_not_forward_authorization(self): api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" signed_url = "https://results-receiver.example/job.log?signature=secret" - redirect = AUDIT.urllib.error.HTTPError( + redirect = AUDIT.urllib_error.HTTPError( api_url, 302, "Found", {"Location": signed_url}, None) - authenticated_opener = mock.Mock() + authenticated_opener = FakeOpener() authenticated_opener.open.side_effect = redirect + build_opener = CallRecorder(return_value=authenticated_opener) + signed_open = CallRecorder(return_value=io.BytesIO(b"job log")) - with mock.patch.object( - AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ - mock.patch.object( - AUDIT.urllib.request, "urlopen", return_value=io.BytesIO(b"job log")) \ - as signed_open: + with patch_attribute(AUDIT.urllib_request, "build_opener", build_opener), \ + patch_attribute(AUDIT.urllib_request, "urlopen", signed_open): payload = AUDIT.request_bytes(api_url, "github-token") self.assertEqual(b"job log", payload) - authenticated_request = authenticated_opener.open.call_args.args[0] + authenticated_request = authenticated_opener.open.calls[0][0][0] self.assertEqual("Bearer github-token", authenticated_request.get_header("Authorization")) - signed_request = signed_open.call_args.args[0] - self.assertEqual(signed_url, signed_request.full_url) + signed_request = signed_open.calls[0][0][0] + self.assertEqual(signed_url, request_url(signed_request)) self.assertIsNone(signed_request.get_header("Authorization")) self.assertIsNone(signed_request.get_header("X-GitHub-Api-Version")) def test_job_log_redirect_requires_location(self): api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" - authenticated_opener = mock.Mock() - authenticated_opener.open.side_effect = AUDIT.urllib.error.HTTPError( + authenticated_opener = FakeOpener() + authenticated_opener.open.side_effect = AUDIT.urllib_error.HTTPError( api_url, 302, "Found", {}, None) - with mock.patch.object( - AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ - self.assertRaisesRegex(AUDIT.JobLogRedirectError, "Location"): + with patch_attribute( + AUDIT.urllib_request, "build_opener", + CallRecorder(return_value=authenticated_opener)), \ + assert_raises_regex(self, AUDIT.JobLogRedirectError, "Location"): AUDIT.request_bytes(api_url, "github-token") def test_job_log_redirect_rejects_non_https_location(self): api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" - authenticated_opener = mock.Mock() - authenticated_opener.open.side_effect = AUDIT.urllib.error.HTTPError( + authenticated_opener = FakeOpener() + authenticated_opener.open.side_effect = AUDIT.urllib_error.HTTPError( api_url, 302, "Found", {"Location": "http://logs.example/job.log"}, None) - with mock.patch.object( - AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ - self.assertRaisesRegex(AUDIT.JobLogRedirectError, "HTTPS"): + with patch_attribute( + AUDIT.urllib_request, "build_opener", + CallRecorder(return_value=authenticated_opener)), \ + assert_raises_regex(self, AUDIT.JobLogRedirectError, "HTTPS"): AUDIT.request_bytes(api_url, "github-token") def test_job_log_endpoint_rejects_non_redirect_response(self): api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" - authenticated_opener = mock.Mock() + authenticated_opener = FakeOpener() authenticated_opener.open.return_value = io.BytesIO(b"unexpected direct response") - with mock.patch.object( - AUDIT.urllib.request, "build_opener", return_value=authenticated_opener), \ - self.assertRaisesRegex(AUDIT.JobLogRedirectError, "expected redirect"): + with patch_attribute( + AUDIT.urllib_request, "build_opener", + CallRecorder(return_value=authenticated_opener)), \ + assert_raises_regex(self, AUDIT.JobLogRedirectError, "expected redirect"): AUDIT.request_bytes(api_url, "github-token") def test_download_logs_preserves_partial_results_after_bad_redirect(self): @@ -165,12 +255,12 @@ def test_download_logs_preserves_partial_results_after_bad_redirect(self): {"id": 2, "name": "package-tests (340, false)", "status": "completed"}, ]} bad_redirect = AUDIT.JobLogRedirectError("missing redirect location") - with mock.patch.object(AUDIT, "request_json", return_value=jobs), \ - mock.patch.object( - AUDIT, "request_bytes", - side_effect=[b"first job log", bad_redirect, bad_redirect, - bad_redirect, bad_redirect]) as request_bytes, \ - mock.patch.object(AUDIT.time, "sleep"): + request_bytes = CallRecorder( + side_effect=[b"first job log", bad_redirect, bad_redirect, + bad_redirect, bad_redirect]) + with patch_attribute(AUDIT, "request_json", CallRecorder(return_value=jobs)), \ + patch_attribute(AUDIT, "request_bytes", request_bytes), \ + patch_attribute(AUDIT.time, "sleep", CallRecorder()): logs, failures = AUDIT.download_logs( "https://api.github.com", "NVIDIA/cudf-spark", "run-id", "token", AUDIT.DEFAULT_JOB_PATTERN) @@ -189,40 +279,50 @@ def test_summary_reports_incomplete_collection(self): def test_main_fails_when_findings_are_present(self): log = ( - "/workspace/sql-plugin/src/main/scala/Test.scala:42: " + u"/workspace/sql-plugin/src/main/scala/Test.scala:42: " "[deprecation @ example.Test.run | " "origin=ai.rapids.cudf.ColumnView.oldApi | version=] deprecated\n" ) - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - (root / "package-tests.log").write_text(log, encoding="utf-8") - result = AUDIT.main([ - "--logs-dir", str(root), - "--raw-report", str(root / "report.json"), - ]) + with temporary_directory() as temp_dir: + log_path = os.path.join(temp_dir, "package-tests.log") + with io.open(log_path, "w", encoding="utf-8") as log_file: + log_file.write(log) + with captured_stdout(): + result = AUDIT.main([ + "--logs-dir", temp_dir, + "--raw-report", os.path.join(temp_dir, "report.json"), + ]) self.assertEqual(1, result) def test_main_succeeds_when_audit_is_clean(self): - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - (root / "package-tests.log").write_text("clean build\n", encoding="utf-8") - result = AUDIT.main([ - "--logs-dir", str(root), - "--raw-report", str(root / "report.json"), - ]) + with temporary_directory() as temp_dir: + log_path = os.path.join(temp_dir, "package-tests.log") + summary_path = os.path.join(temp_dir, "summary.md") + with io.open(log_path, "w", encoding="utf-8") as log_file: + log_file.write(u"clean build\n") + with captured_stdout(): + result = AUDIT.main([ + "--logs-dir", temp_dir, + "--summary", summary_path, + "--raw-report", os.path.join(temp_dir, "report.json"), + ]) + with io.open(summary_path, "r", encoding="utf-8") as summary_file: + self.assertIn("No compiler deprecation diagnostics", summary_file.read()) self.assertEqual(0, result) def test_main_fails_when_log_collection_is_incomplete(self): - with tempfile.TemporaryDirectory() as temp_dir, \ - mock.patch.object( + with temporary_directory() as temp_dir, \ + patch_attribute( AUDIT, "download_logs", - return_value=({}, ["package-tests: log unavailable"])), \ - mock.patch.dict(AUDIT.os.environ, {"GITHUB_TOKEN": "token"}): - result = AUDIT.main([ - "--repository", "NVIDIA/cudf-spark", - "--run-id", "123", - "--raw-report", str(Path(temp_dir) / "report.json"), - ]) + CallRecorder(return_value=({}, ["package-tests: log unavailable"]))), \ + patch_environment({"GITHUB_TOKEN": "token"}): + with captured_stdout(): + result = AUDIT.main([ + "--repository", "NVIDIA/cudf-spark", + "--run-id", "123", + "--logs-dir", "", + "--raw-report", os.path.join(temp_dir, "report.json"), + ]) self.assertEqual(1, result) def test_annotation_property_escaping(self): From 46edaea14413837b6a2dc2c7d19c7e2d3ea0be7e Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Tue, 1 Sep 2026 12:17:33 -0700 Subject: [PATCH 4/5] Launch Jython from the plugin classpath Signed-off-by: Gera Shegalov --- pom.xml | 9 ++++++--- scala2.13/pom.xml | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index ff8be39f99d..41a757ba759 100644 --- a/pom.xml +++ b/pom.xml @@ -2013,12 +2013,14 @@ This will force full Scala code rebuild in downstream modules. run - - @@ -2077,7 +2079,8 @@ This will force full Scala code rebuild in downstream modules. - diff --git a/scala2.13/pom.xml b/scala2.13/pom.xml index 238f6916534..c1c82abe6da 100644 --- a/scala2.13/pom.xml +++ b/scala2.13/pom.xml @@ -2013,12 +2013,14 @@ This will force full Scala code rebuild in downstream modules. run - - @@ -2077,7 +2079,8 @@ This will force full Scala code rebuild in downstream modules. - From 7781efef90a92e5faaae9412957562e80e79ca3e Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Tue, 1 Sep 2026 12:50:44 -0700 Subject: [PATCH 5/5] Keep deprecation audit on root POM Signed-off-by: Gera Shegalov --- pom.xml | 7 +++++-- scala2.13/pom.xml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 41a757ba759..056c7744f95 100644 --- a/pom.xml +++ b/pom.xml @@ -2007,6 +2007,8 @@ This will force full Scala code rebuild in downstream modules. false + + nvidia-deprecation-audit none @@ -2024,14 +2026,15 @@ This will force full Scala code rebuild in downstream modules. fork="true" failonerror="true" dir="${project.basedir}"> - + - + + false + +