diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 2afb1eebb7d..54c3d2b54e2 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -36,6 +36,7 @@ env: -Drapids.secondaryCacheDir=$HOME/.m2/repository/.sbt/1.0/zinc/org.scala-sbt permissions: + actions: read contents: read jobs: @@ -430,3 +431,31 @@ jobs: fi } done + + nvidia-deprecation-audit: + name: NVIDIA deprecation audit (optional) + if: ${{ always() }} + needs: + - package-tests + - package-tests-scala213 + - verify-213-modules + - verify-all-212-modules + - install-modules + runs-on: ubuntu-latest + steps: + - uses: NVIDIA/spark-rapids-common/checkout@main + + - name: Collect compiler deprecations from matrix logs + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + mvn --batch-mode -N antrun:run@nvidia-deprecation-audit \ + -Ddeprecation.audit.rawReport="$RUNNER_TEMP/nvidia-deprecation-audit.json" + + - name: Upload deprecation report + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: nvidia-deprecation-audit + path: ${{ runner.temp }}/nvidia-deprecation-audit.json + if-no-files-found: ignore diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf2263c228b..5b58c20eee2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -235,6 +235,24 @@ or similarly ./build/buildall --rebuild-dist-only --option="-Ddist.jar.compress=false -Drapids.jni.unpack.skip" ``` +### Cross-repository API deprecations + +API replacements in cuDF Java, cudf-spark-jni, and cudf-spark-private must allow cudf-spark time to +consume a published artifact before the old entry point becomes deprecated. Introduce the +replacement first while the old method remains supported and delegates to the same implementation. +After the updated snapshot is available, migrate cudf-spark callers in a separate change. Add the +deprecation annotation only after known callers have migrated, and retain the compatibility entry +point for at least one more release before removal. + +Scala deprecations originating in NVIDIA-owned `ai.rapids.cudf`, `com.nvidia.spark.rapids`, and +`org.apache.spark.sql.rapids` APIs are reported as compiler information instead of fatal warnings +during this migration window. The same applies to cudf-spark-private's +`org.apache.spark.sql.execution.aggregate.PartialAggUtils` bridge; other APIs in Apache Spark +namespaces are not exempt. Deprecations from other dependencies remain build errors. The +optional NVIDIA deprecation audit in pull requests collects these diagnostics across the Maven +build matrix. Findings or incomplete log collection fail the audit check so contributors inspect +the result, but the audit is not a required build check; build-job results remain authoritative. + ## Code contributions ### Source code layout diff --git a/pom.xml b/pom.xml index 380f2e1e1f0..ce27e84dd97 100644 --- a/pom.xml +++ b/pom.xml @@ -1152,6 +1152,8 @@ 4.9.10 3.1.1 3.3.0 + 2.7.3 + ${project.build.directory}/nvidia-deprecation-audit.json 2.0.2 30.0-jre 2.0.0 @@ -1618,7 +1620,7 @@ This will force full Scala code rebuild in downstream modules. org.python jython-standalone - 2.7.3 + ${jython.version} net.sourceforge.pmd @@ -1742,6 +1744,18 @@ This will force full Scala code rebuild in downstream modules. -Wconf:cat=unused-privates:e -Wunused:imports,locals,patvars,privates --> + + -Wconf:cat=deprecation&origin=ai\.rapids\.cudf\..*:iv + -Wconf:cat=deprecation&origin=com\.nvidia\.spark\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.execution\.aggregate\.PartialAggUtils([.$].*|$):iv ${scala.javac.args} false + + + + nvidia-deprecation-audit + none + run + + + + + + + + + + + + + + + + + + 4.9.10 3.1.1 3.3.0 + 2.7.3 + ${project.build.directory}/nvidia-deprecation-audit.json 2.0.2 30.0-jre 2.0.0 @@ -1618,7 +1620,7 @@ This will force full Scala code rebuild in downstream modules. org.python jython-standalone - 2.7.3 + ${jython.version} net.sourceforge.pmd @@ -1742,6 +1744,18 @@ This will force full Scala code rebuild in downstream modules. -Wconf:cat=unused-privates:e -Wunused:imports,locals,patvars,privates + + -Wconf:cat=deprecation&origin=ai\.rapids\.cudf\..*:iv + -Wconf:cat=deprecation&origin=com\.nvidia\.spark\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.rapids\..*:iv + -Wconf:cat=deprecation&origin=org\.apache\.spark\.sql\.execution\.aggregate\.PartialAggUtils([.$].*|$):iv ${scala.javac.args} @@ -1992,6 +2006,34 @@ This will force full Scala code rebuild in downstream modules. false + + (?:[A-Za-z]:)?[^\s\[\]]+?\.(?:scala|java))" + r"(?::(?P\d+)|:\[(?P\d+),\d+\])" +) +DEPRECATION = re.compile(r"\bdeprecated\b", re.IGNORECASE) +ORIGIN = re.compile(r"\borigin(?:=|:)\s*(?P[\w.$]+)") +DEFAULT_JOB_PATTERN = ( + r"^(?:package-tests(?:-scala213)?|verify-213-modules|" + r"verify-all-212-modules|install-modules)(?:\s|$)" +) +NVIDIA_ORIGIN_PREFIXES = ( + "ai.rapids.cudf.", + "com.nvidia.spark.rapids.", + "org.apache.spark.sql.rapids.", +) +NVIDIA_ORIGIN_SYMBOLS = ( + "org.apache.spark.sql.execution.aggregate.PartialAggUtils", +) + + +def is_nvidia_origin(origin): + if origin.startswith(NVIDIA_ORIGIN_PREFIXES): + return True + return any( + origin == symbol or origin.startswith((symbol + ".", symbol + "$")) + for symbol in NVIDIA_ORIGIN_SYMBOLS + ) + + +BAD_ZIP_ERROR = getattr(zipfile, "BadZipFile", zipfile.BadZipfile) + + +class Finding(object): + def __init__(self, path, line, message, origin="", jobs=None): + self.path = path + self.line = line + self.message = message + self.origin = origin + self.jobs = set(jobs or ()) + + @property + def owner(self): + if is_nvidia_origin(self.origin): + return "NVIDIA" + return "third-party/unknown" + + def key(self): + diagnostic = self.origin or self.message + return self.path, self.line, diagnostic + + +def clean_line(line): + return ANSI_ESCAPE.sub("", line).rstrip() + + +def read_text(path): + with io.open(path, "r", encoding="utf-8", errors="replace") as input_file: + return input_file.read() + + +def normalize_path(path, repo_root): + candidate = os.path.normpath(path) + if not os.path.isabs(candidate): + return candidate.replace(os.sep, "/") + root = os.path.realpath(repo_root) + resolved = os.path.realpath(candidate) + relative = os.path.relpath(resolved, root) + if relative != os.pardir and not relative.startswith(os.pardir + os.sep): + return relative.replace(os.sep, "/") + parts = candidate.split(os.sep) + for index in range(len(parts)): + suffix = os.path.join(*parts[index:]) + if os.path.exists(os.path.join(root, suffix)): + return suffix.replace(os.sep, "/") + return candidate.replace(os.sep, "/") + + +def parse_log(text, job_name, repo_root="."): + lines = [clean_line(line) for line in text.splitlines()] + findings = [] + for index, line in enumerate(lines): + if not DEPRECATION.search(line): + continue + location = SOURCE_LOCATION.search(line) + if location is None: + for previous in reversed(lines[max(0, index - 3):index]): + location = SOURCE_LOCATION.search(previous) + if location is not None: + break + if location is None: + continue + origin = "" + for context in lines[index:min(len(lines), index + 6)]: + origin_match = ORIGIN.search(context) + if origin_match is not None: + origin = origin_match.group("origin") + break + line_number = location.group("line") or location.group("bracket_line") + message = re.sub(r"^.*?\.(?:scala|java)(?::\d+|:\[\d+,\d+\])\s*:?[ ]*", "", line) + findings.append(Finding( + path=normalize_path(location.group("path"), repo_root), + line=int(line_number), + message=message.strip() or line.strip(), + origin=origin, + jobs={job_name}, + )) + return findings + + +def merge_findings(findings): + merged = {} + for finding in findings: + existing = merged.get(finding.key()) + if existing is None: + merged[finding.key()] = finding + else: + existing.jobs.update(finding.jobs) + return sorted( + merged.values(), key=lambda finding: (finding.path, finding.line, finding.message)) + + +def request_json(url, token): + request = urllib_request.Request(url, headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer {0}".format(token), + "X-GitHub-Api-Version": "2022-11-28", + }) + response = urllib_request.urlopen(request, timeout=30) + try: + return json.load(response) + finally: + response.close() + + +class JobLogRedirectError(RuntimeError): + """The GitHub job-log endpoint returned an unsafe or malformed redirect.""" + + +class NoRedirectHandler(urllib_request.HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +def request_bytes(url, token): + """Download a GitHub API resource without forwarding credentials on its redirect.""" + request = urllib_request.Request(url, headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer {0}".format(token), + "X-GitHub-Api-Version": "2022-11-28", + }) + opener = urllib_request.build_opener(NoRedirectHandler()) + try: + response = opener.open(request, timeout=30) + try: + raise JobLogRedirectError( + "GitHub job-log endpoint did not return the expected redirect") + finally: + response.close() + except urllib_error.HTTPError as error: + if error.code != 302: + error.close() + raise + headers = getattr(error, "headers", None) + if headers is None: + headers = error.hdrs + location = headers.get("Location") + error.close() + if not location: + raise JobLogRedirectError( + "GitHub job-log redirect did not include a Location header") + parsed_location = urllib_parse.urlsplit(location) + if parsed_location.scheme != "https" or not parsed_location.netloc: + raise JobLogRedirectError( + "GitHub job-log redirect must use an absolute HTTPS URL") + + # The redirect is a short-lived signed URL. It authorizes itself, so use a fresh request + # without the repository-scoped GitHub token or GitHub-specific API headers. + signed_request = urllib_request.Request(location) + response = urllib_request.urlopen(signed_request, timeout=30) + try: + return response.read() + finally: + response.close() + + +def decode_job_log(payload): + if payload.startswith(b"PK"): + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + return u"\n".join( + archive.read(name).decode("utf-8", errors="replace") + for name in archive.namelist() + if not name.endswith("/") + ) + return payload.decode("utf-8", errors="replace") + + +def download_logs(api_url, repository, run_id, token, job_pattern): + matcher = re.compile(job_pattern) + jobs = [] + page = 1 + while True: + result = request_json( + "{0}/repos/{1}/actions/runs/{2}/jobs?per_page=100&page={3}".format( + api_url, repository, run_id, page), + token, + ) + page_jobs = result.get("jobs", []) + jobs.extend(page_jobs) + if len(page_jobs) < 100: + break + page += 1 + + logs = {} + failures = [] + for job in jobs: + name = job.get("name", "") + if job.get("status") != "completed" or matcher.search(name) is None: + continue + error = None + for delay in (0, 1, 2, 4): + if delay: + time.sleep(delay) + try: + payload = request_bytes( + "{0}/repos/{1}/actions/jobs/{2}/logs".format( + api_url, repository, job["id"]), token) + logs[name] = decode_job_log(payload) + error = None + break + except (OSError, urllib_error.HTTPError, BAD_ZIP_ERROR, + JobLogRedirectError) as caught: + error = caught + if error is not None: + failures.append(u"{0}: {1}".format(name, error)) + if not logs and not failures: + failures.append("no completed build-matrix job logs matched the configured job pattern") + return logs, failures + + +def markdown_escape(value): + return value.replace("|", "\\|").replace("\n", " ") + + +def render_summary(findings, failures): + lines = [u"## NVIDIA deprecation audit", u""] + if findings: + lines.extend([ + u"Found {0} unique compiler deprecation diagnostic(s).".format(len(findings)), + u"", + u"| Owner | Location | Deprecated API | Matrix jobs |", + u"| --- | --- | --- | --- |", + ]) + for finding in findings[:200]: + location = u"`{0}:{1}`".format(finding.path, finding.line) + api = finding.origin or finding.message + jobs = u", ".join(sorted(finding.jobs)) + lines.append( + u"| {0} | {1} | `{2}` | {3} |".format( + finding.owner, location, markdown_escape(api), markdown_escape(jobs)) + ) + if len(findings) > 200: + lines.extend([ + u"", + u"Report truncated; see the raw artifact for all {0} findings.".format( + len(findings)), + ]) + else: + lines.append(u"No compiler deprecation diagnostics were found in the selected matrix jobs.") + if failures: + lines.extend([u"", u"### Incomplete log collection", u""]) + lines.extend(u"- {0}".format(markdown_escape(failure)) for failure in failures) + lines.extend([ + u"", + u"This check is optional. Findings or incomplete log collection fail only this audit " + "check; build-job results remain authoritative.", + u"", + ]) + return u"\n".join(lines) + + +def command_escape(value): + return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def command_property_escape(value): + return command_escape(value).replace(":", "%3A").replace(",", "%2C") + + +def emit_annotations(findings): + for finding in findings[:50]: + message = finding.origin or finding.message + print( + u"::warning file={0},line={1},title=NVIDIA deprecation::{2}".format( + command_property_escape(finding.path), finding.line, command_escape(message)) + ) + if len(findings) > 50: + print(u"::warning title=NVIDIA deprecation::Only 50 of {0} findings were annotated".format( + len(findings))) + + +def write_raw_report(path, findings, failures): + report = { + "findings": [ + { + "owner": finding.owner, + "path": finding.path, + "line": finding.line, + "message": finding.message, + "origin": finding.origin, + "jobs": sorted(finding.jobs), + } + for finding in findings + ], + "log_collection_failures": failures, + } + with io.open(path, "w", encoding="utf-8") as report_file: + report_file.write(json.dumps(report, indent=2, ensure_ascii=False) + u"\n") + + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY")) + parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID")) + parser.add_argument( + "--api-url", default=os.environ.get("GITHUB_API_URL", "https://api.github.com")) + parser.add_argument("--job-pattern", default=DEFAULT_JOB_PATTERN) + parser.add_argument( + "--logs-dir", default=os.environ.get("DEPRECATION_AUDIT_LOGS_DIR"), + help="Parse local *.log files instead of downloading job logs") + parser.add_argument("--repo-root", default=".") + parser.add_argument("--summary", default=os.environ.get("GITHUB_STEP_SUMMARY")) + parser.add_argument("--raw-report", default="nvidia-deprecation-audit.json") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + failures = [] + try: + if args.logs_dir: + logs = { + os.path.splitext(os.path.basename(path))[0]: read_text(path) + for path in glob.glob(os.path.join(args.logs_dir, "*.log")) + } + else: + token = os.environ.get("GITHUB_TOKEN") + if not token or not args.repository or not args.run_id: + raise ValueError("GITHUB_TOKEN, repository, and run ID are required") + logs, failures = download_logs( + args.api_url, args.repository, args.run_id, token, args.job_pattern) + findings = merge_findings( + finding + for job_name, log in logs.items() + for finding in parse_log(log, job_name, args.repo_root) + ) + except Exception as error: # Report operational errors through this optional check. + findings = [] + failures.append(u"audit failed: {0}".format(error)) + + summary = render_summary(findings, failures) + print(summary) + emit_annotations(findings) + if args.summary: + with io.open(args.summary, "a", encoding="utf-8") as summary_file: + summary_file.write(summary) + report_failed = False + try: + write_raw_report(args.raw_report, findings, failures) + except (IOError, OSError) as error: + report_failed = True + print(u"::warning title=NVIDIA deprecation audit::Could not write raw report: {0}".format( + error)) + return 1 if findings or failures or report_failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_deprecation_audit.py b/scripts/tests/test_deprecation_audit.py new file mode 100644 index 00000000000..b6e20f820a0 --- /dev/null +++ b/scripts/tests/test_deprecation_audit.py @@ -0,0 +1,334 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import contextlib +import io +import os +import shutil +import sys +import tempfile +import unittest +import zipfile + + +SCRIPT = os.path.join(os.path.dirname(os.path.dirname(__file__)), "deprecation_audit.py") +try: + import importlib.util + SPEC = importlib.util.spec_from_file_location("deprecation_audit", SCRIPT) + AUDIT = importlib.util.module_from_spec(SPEC) + sys.modules[SPEC.name] = AUDIT + SPEC.loader.exec_module(AUDIT) +except ImportError: # Jython 2.7 / Python 2.7 + import imp + AUDIT = imp.load_source("deprecation_audit", SCRIPT) + + +class CallRecorder(object): + def __init__(self, return_value=None, side_effect=None): + self.return_value = return_value + self.side_effect = side_effect + self.calls = [] + + @property + def call_count(self): + return len(self.calls) + + def __call__(self, *args, **kwargs): + self.calls.append((args, kwargs)) + effect = self.side_effect + if isinstance(effect, list): + effect = effect.pop(0) + if isinstance(effect, BaseException): + raise effect + if callable(effect): + return effect(*args, **kwargs) + return self.return_value if effect is None else effect + + +class FakeOpener(object): + def __init__(self): + self.open = CallRecorder() + + +class OutputSink(object): + def __init__(self): + self.parts = [] + + def write(self, value): + self.parts.append(value) + + def flush(self): + pass + + +@contextlib.contextmanager +def patch_attribute(target, name, value): + original = getattr(target, name) + setattr(target, name, value) + try: + yield value + finally: + setattr(target, name, original) + + +@contextlib.contextmanager +def patch_environment(updates): + original = dict(os.environ) + os.environ.update(updates) + try: + yield + finally: + os.environ.clear() + os.environ.update(original) + + +@contextlib.contextmanager +def temporary_directory(): + path = tempfile.mkdtemp() + try: + yield path + finally: + shutil.rmtree(path) + + +@contextlib.contextmanager +def captured_stdout(): + output = OutputSink() + with patch_attribute(sys, "stdout", output): + yield output + + +def request_url(request): + return request.get_full_url() + + +def assert_raises_regex(test_case, exception, pattern): + method = getattr(test_case, "assertRaisesRegex", None) + if method is None: + method = test_case.assertRaisesRegexp + return method(exception, pattern) + + +class DeprecationAuditSuite(unittest.TestCase): + def test_parses_scala_verbose_deprecation(self): + log = ( + "[INFO] /workspace/sql-plugin/src/main/scala/Test.scala:42: " + "[deprecation @ example.Test.run | " + "origin=ai.rapids.cudf.ColumnView.oldApi | version=] " + "method oldApi in class ColumnView is deprecated\n" + ) + findings = AUDIT.parse_log(log, "package-tests (330)") + self.assertEqual(1, len(findings)) + self.assertEqual(42, findings[0].line) + self.assertEqual("ai.rapids.cudf.ColumnView.oldApi", findings[0].origin) + self.assertEqual("NVIDIA", findings[0].owner) + + def test_parses_maven_bracket_location(self): + log = """ +[WARNING] /workspace/src/main/java/Test.java:[17,9] oldApi() has been deprecated +""" + findings = AUDIT.parse_log(log, "verify-all-212-modules (330, 17)") + self.assertEqual(17, findings[0].line) + self.assertEqual("third-party/unknown", findings[0].owner) + + def test_reads_scala_213_origin_from_following_line(self): + log = """ +[INFO] /workspace/sql-plugin/src/main/scala/Test.scala:42: method oldApi is deprecated +Applicable -Wconf filters: cat=deprecation, origin=com.nvidia.spark.rapids.jni.Api.oldApi +""" + findings = AUDIT.parse_log(log, "package-tests-scala213 (350)") + self.assertEqual("com.nvidia.spark.rapids.jni.Api.oldApi", findings[0].origin) + self.assertEqual("NVIDIA", findings[0].owner) + + def test_classifies_all_advisory_origins_as_nvidia(self): + origins = ( + "ai.rapids.cudf.ColumnView.oldApi", + "com.nvidia.spark.rapids.optimizer.OptimizerConf.oldApi", + "org.apache.spark.sql.rapids.internal.PrivateRapidsConfs.oldApi", + "org.apache.spark.sql.execution.aggregate.PartialAggUtils.oldApi", + "org.apache.spark.sql.execution.aggregate.PartialAggUtils$Helper.oldApi", + ) + for origin in origins: + finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) + self.assertEqual("NVIDIA", finding.owner, origin) + + def test_partial_agg_utils_owner_match_has_symbol_boundary(self): + origins = ( + "org.apache.spark.sql.execution.aggregate.PartialAggUtilsNeighbor.oldApi", + "org.apache.spark.sql.execution.aggregate.SparkApi.oldApi", + "org.example.fixture.ThirdPartyApi.oldApi", + ) + for origin in origins: + finding = AUDIT.Finding("Test.scala", 1, "deprecated", origin) + self.assertEqual("third-party/unknown", finding.owner, origin) + + def test_ignores_non_source_deprecation_text(self): + log = "[WARNING] This build plugin uses a deprecated Maven feature\n" + self.assertEqual([], AUDIT.parse_log(log, "install-modules (3.9.3)")) + + def test_merges_same_finding_across_matrix_jobs(self): + first = AUDIT.Finding( + "Test.scala", 1, "[deprecation] old is deprecated", "ai.rapids.cudf.Api.old", + {"330"}) + second = AUDIT.Finding( + "Test.scala", 1, "old is deprecated", "ai.rapids.cudf.Api.old", {"400"}) + merged = AUDIT.merge_findings([first, second]) + self.assertEqual({"330", "400"}, merged[0].jobs) + + def test_decodes_zip_job_log(self): + payload = io.BytesIO() + with zipfile.ZipFile(payload, "w") as archive: + archive.writestr("job/step.txt", "deprecated output") + self.assertEqual("deprecated output", AUDIT.decode_job_log(payload.getvalue())) + + def test_job_log_redirect_does_not_forward_authorization(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + signed_url = "https://results-receiver.example/job.log?signature=secret" + redirect = AUDIT.urllib_error.HTTPError( + api_url, 302, "Found", {"Location": signed_url}, None) + authenticated_opener = FakeOpener() + authenticated_opener.open.side_effect = redirect + build_opener = CallRecorder(return_value=authenticated_opener) + signed_open = CallRecorder(return_value=io.BytesIO(b"job log")) + + with patch_attribute(AUDIT.urllib_request, "build_opener", build_opener), \ + patch_attribute(AUDIT.urllib_request, "urlopen", signed_open): + payload = AUDIT.request_bytes(api_url, "github-token") + + self.assertEqual(b"job log", payload) + authenticated_request = authenticated_opener.open.calls[0][0][0] + self.assertEqual("Bearer github-token", + authenticated_request.get_header("Authorization")) + signed_request = signed_open.calls[0][0][0] + self.assertEqual(signed_url, request_url(signed_request)) + self.assertIsNone(signed_request.get_header("Authorization")) + self.assertIsNone(signed_request.get_header("X-GitHub-Api-Version")) + + def test_job_log_redirect_requires_location(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + authenticated_opener = FakeOpener() + authenticated_opener.open.side_effect = AUDIT.urllib_error.HTTPError( + api_url, 302, "Found", {}, None) + with patch_attribute( + AUDIT.urllib_request, "build_opener", + CallRecorder(return_value=authenticated_opener)), \ + assert_raises_regex(self, AUDIT.JobLogRedirectError, "Location"): + AUDIT.request_bytes(api_url, "github-token") + + def test_job_log_redirect_rejects_non_https_location(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + authenticated_opener = FakeOpener() + authenticated_opener.open.side_effect = AUDIT.urllib_error.HTTPError( + api_url, 302, "Found", {"Location": "http://logs.example/job.log"}, None) + with patch_attribute( + AUDIT.urllib_request, "build_opener", + CallRecorder(return_value=authenticated_opener)), \ + assert_raises_regex(self, AUDIT.JobLogRedirectError, "HTTPS"): + AUDIT.request_bytes(api_url, "github-token") + + def test_job_log_endpoint_rejects_non_redirect_response(self): + api_url = "https://api.github.com/repos/NVIDIA/cudf-spark/actions/jobs/123/logs" + authenticated_opener = FakeOpener() + authenticated_opener.open.return_value = io.BytesIO(b"unexpected direct response") + with patch_attribute( + AUDIT.urllib_request, "build_opener", + CallRecorder(return_value=authenticated_opener)), \ + assert_raises_regex(self, AUDIT.JobLogRedirectError, "expected redirect"): + AUDIT.request_bytes(api_url, "github-token") + + def test_download_logs_preserves_partial_results_after_bad_redirect(self): + jobs = {"jobs": [ + {"id": 1, "name": "package-tests (330, false)", "status": "completed"}, + {"id": 2, "name": "package-tests (340, false)", "status": "completed"}, + ]} + bad_redirect = AUDIT.JobLogRedirectError("missing redirect location") + request_bytes = CallRecorder( + side_effect=[b"first job log", bad_redirect, bad_redirect, + bad_redirect, bad_redirect]) + with patch_attribute(AUDIT, "request_json", CallRecorder(return_value=jobs)), \ + patch_attribute(AUDIT, "request_bytes", request_bytes), \ + patch_attribute(AUDIT.time, "sleep", CallRecorder()): + logs, failures = AUDIT.download_logs( + "https://api.github.com", "NVIDIA/cudf-spark", "run-id", "token", + AUDIT.DEFAULT_JOB_PATTERN) + + self.assertEqual({"package-tests (330, false)": "first job log"}, logs) + self.assertEqual(5, request_bytes.call_count) + self.assertEqual(1, len(failures)) + self.assertIn("package-tests (340, false)", failures[0]) + self.assertIn("missing redirect location", failures[0]) + + def test_summary_reports_incomplete_collection(self): + summary = AUDIT.render_summary([], ["package-tests: log unavailable"]) + self.assertIn("No compiler deprecation diagnostics", summary) + self.assertIn("Incomplete log collection", summary) + self.assertIn("optional", summary) + + def test_main_fails_when_findings_are_present(self): + log = ( + u"/workspace/sql-plugin/src/main/scala/Test.scala:42: " + "[deprecation @ example.Test.run | " + "origin=ai.rapids.cudf.ColumnView.oldApi | version=] deprecated\n" + ) + with temporary_directory() as temp_dir: + log_path = os.path.join(temp_dir, "package-tests.log") + with io.open(log_path, "w", encoding="utf-8") as log_file: + log_file.write(log) + with captured_stdout(): + result = AUDIT.main([ + "--logs-dir", temp_dir, + "--raw-report", os.path.join(temp_dir, "report.json"), + ]) + self.assertEqual(1, result) + + def test_main_succeeds_when_audit_is_clean(self): + with temporary_directory() as temp_dir: + log_path = os.path.join(temp_dir, "package-tests.log") + summary_path = os.path.join(temp_dir, "summary.md") + with io.open(log_path, "w", encoding="utf-8") as log_file: + log_file.write(u"clean build\n") + with captured_stdout(): + result = AUDIT.main([ + "--logs-dir", temp_dir, + "--summary", summary_path, + "--raw-report", os.path.join(temp_dir, "report.json"), + ]) + with io.open(summary_path, "r", encoding="utf-8") as summary_file: + self.assertIn("No compiler deprecation diagnostics", summary_file.read()) + self.assertEqual(0, result) + + def test_main_fails_when_log_collection_is_incomplete(self): + with temporary_directory() as temp_dir, \ + patch_attribute( + AUDIT, "download_logs", + CallRecorder(return_value=({}, ["package-tests: log unavailable"]))), \ + patch_environment({"GITHUB_TOKEN": "token"}): + with captured_stdout(): + result = AUDIT.main([ + "--repository", "NVIDIA/cudf-spark", + "--run-id", "123", + "--logs-dir", "", + "--raw-report", os.path.join(temp_dir, "report.json"), + ]) + self.assertEqual(1, result) + + def test_annotation_property_escaping(self): + self.assertEqual("path%3Awith%2Cpunctuation", AUDIT.command_property_escape( + "path:with,punctuation")) + + +if __name__ == "__main__": + unittest.main()