diff --git a/.github/workflows/actions-ci.yml b/.github/workflows/actions-ci.yml index ebe358b2f6..8bcef7b2ca 100644 --- a/.github/workflows/actions-ci.yml +++ b/.github/workflows/actions-ci.yml @@ -15,6 +15,10 @@ env: # Used to enable ASAN test dimension. AWSLC_ENABLE_FIPS_ASAN: 1 DEBIAN_FRONTEND: noninteractive + # Set globally so every job routing tests through util/all_tests.go emits + # per-test-case gtest JSON reports. Jobs opt in to surfacing the summary by + # adding a "Test timing summary" step. + GTEST_REPORT_DIR: ${{ github.workspace }}/gtest_reports permissions: contents: read @@ -52,6 +56,9 @@ jobs: - name: Build ${{ env.PACKAGE_NAME }} run: | ./tests/ci/run_posix_tests.sh + - name: Test timing summary + if: always() + run: python3 tests/ci/test_results_summary.py "${{ github.workspace }}/gtest_reports" >> $GITHUB_STEP_SUMMARY macOS-x86-FIPS: if: github.repository_owner == 'aws' @@ -70,6 +77,9 @@ jobs: - name: Build ${{ env.PACKAGE_NAME }} with FIPS mode run: | ./tests/ci/run_fips_tests.sh + - name: Test timing summary + if: always() + run: python3 tests/ci/test_results_summary.py "${{ github.workspace }}/gtest_reports" >> $GITHUB_STEP_SUMMARY macOS-ARM: if: github.repository_owner == 'aws' @@ -89,6 +99,9 @@ jobs: - name: Build ${{ env.PACKAGE_NAME }} run: | ./tests/ci/run_posix_tests.sh + - name: Test timing summary + if: always() + run: python3 tests/ci/test_results_summary.py "${{ github.workspace }}/gtest_reports" >> $GITHUB_STEP_SUMMARY windows: if: github.repository_owner == 'aws' @@ -149,6 +162,9 @@ jobs: - name: Build ${{ env.PACKAGE_NAME }} with FIPS mode run: | ./tests/ci/run_fips_tests.sh + - name: Test timing summary + if: always() + run: python3 tests/ci/test_results_summary.py "${{ github.workspace }}/gtest_reports" >> $GITHUB_STEP_SUMMARY compiler-tests: name: >- diff --git a/tests/ci/test_results_summary.py b/tests/ci/test_results_summary.py new file mode 100644 index 0000000000..04b672165f --- /dev/null +++ b/tests/ci/test_results_summary.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR ISC + +"""Parse gtest JSON reports and write a GitHub Actions step summary.""" + +import glob +import json +import os +import sys + + +def main(): + report_dir = sys.argv[1] if len(sys.argv) > 1 else os.environ.get( + "GTEST_REPORT_DIR", "gtest_reports" + ) + + if not os.path.isdir(report_dir): + print(f"No report directory found at {report_dir}", file=sys.stderr) + sys.exit(0) + + files = sorted(glob.glob(os.path.join(report_dir, "*.json"))) + if not files: + print(f"No JSON reports found in {report_dir}", file=sys.stderr) + sys.exit(0) + + all_tests = [] # (name, seconds, status, binary) + + for f in files: + binary = os.path.basename(f).rsplit("_shard_", 1)[0] + try: + with open(f) as fh: + data = json.load(fh) + except (json.JSONDecodeError, IOError): + continue + + for suite in data.get("testsuites", []): + suite_name = suite.get("name", "") + for test in suite.get("testsuite", []): + # Skip tests that never executed (disabled/suppressed); they + # have no meaningful timing and aren't failures. + if test.get("result") == "SUPPRESSED" or test.get("status") == "NOTRUN": + continue + name = f"{suite_name}.{test.get('name', '')}" + time_str = test.get("time", "0s") + secs = float(time_str.rstrip("s")) if time_str.endswith("s") else float(time_str) + status = "FAIL" if test.get("failures") is not None else "PASS" + all_tests.append((name, secs, status, binary)) + + if not all_tests: + sys.exit(0) + + all_tests.sort(key=lambda x: x[1], reverse=True) + + total = len(all_tests) + failed = sum(1 for t in all_tests if t[2] != "PASS") + + lines = [] + lines.append("## 🧪 Test Timing Summary\n") + lines.append(f"**{total}** tests across {len(files)} report(s), **{failed}** failed\n") + lines.append("### ⏱️ Top 20 Slowest Tests\n") + lines.append("| Test | Time | Binary |") + lines.append("|------|------|--------|") + for name, secs, status, binary in all_tests[:20]: + if secs >= 60: + time_str = f"{secs/60:.1f}m" + else: + time_str = f"{secs:.1f}s" + icon = "" if status == "PASS" else " ❌" + lines.append(f"| `{name}`{icon} | {time_str} | {binary} |") + + lines.append("") + sys.stdout.write("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/util/all_tests.go b/util/all_tests.go index d12b9963e4..aa027505e6 100644 --- a/util/all_tests.go +++ b/util/all_tests.go @@ -19,6 +19,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "syscall" "time" @@ -242,6 +243,9 @@ var ( errTestHanging = errors.New("test hangs without exiting") ) +// reportSeq uniquely identifies each gtest JSON report file. +var reportSeq int64 + func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) { prog := filepath.Join(*buildDir, test.Cmd[0]) args := append([]string{}, test.Cmd[1:]...) @@ -285,6 +289,23 @@ func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) { cmd.Env = append(cmd.Env, fmt.Sprintf("GTEST_SHARD_INDEX=%d", test.shard)) cmd.Env = append(cmd.Env, fmt.Sprintf("GTEST_TOTAL_SHARDS=%d", test.numShards)) } + // Write per-test-case gtest JSON report if report dir is set. + // Report generation is a diagnostic aid, so on error we warn and skip the report + // rather than failing the test run. + if reportDir := os.Getenv("GTEST_REPORT_DIR"); reportDir != "" { + if err := os.MkdirAll(reportDir, 0755); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not create GTEST_REPORT_DIR %q: %v; skipping timing report\n", reportDir, err) + } else { + binName := filepath.Base(prog) + reportFile := filepath.Join(reportDir, fmt.Sprintf("%s_shard_%d_%d_%d.json", + binName, test.shard, os.Getpid(), atomic.AddInt64(&reportSeq, 1))) + if cmd.Env == nil { + cmd.Env = make([]string, len(os.Environ())) + copy(cmd.Env, os.Environ()) + } + cmd.Env = append(cmd.Env, fmt.Sprintf("GTEST_OUTPUT=json:%s", reportFile)) + } + } var outBuf bytes.Buffer cmd.Stdout = &outBuf cmd.Stderr = &outBuf