ecosystem-tests.yml

name: Ecosystem Tests

on:
  pull_request:
    paths:
      - 'crates/**'
      - 'ecosystem/**'
      - '.github/workflows/ecosystem-tests.yml'
      - 'Cargo.toml'
      - 'Cargo.lock'
  push:
    branches: [main]
  workflow_dispatch:

permissions:
  pull-requests: write
  issues: write

jobs:
  ecosystem-test:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
        with:
          submodules: recursive

      - name: Setup Rust
        run: rustup show

      - name: Cache Rust dependencies
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            target
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-

      - name: Set up uv
        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
        with:
          enable-cache: true
          activate-environment: true
          python-version: 3.12

      - name: Install Python dependencies
        run: |
          uv sync --all-groups

      - name: Build cribo (release mode)
        run: cargo build --release --bin cribo

      - name: Run ecosystem tests
        continue-on-error: true
        run: |
          # Prefer the prebuilt binary for the runner
          export CARGO_BIN_EXE_cribo="$PWD/target/release/cribo"
          echo "CARGO_BIN_EXE_cribo=$CARGO_BIN_EXE_cribo" >> "$GITHUB_ENV"

          # Run all ecosystem tests with pytest
          # --tb=short for shorter tracebacks
          # -v for verbose output
          # -ra for summary of all test outcomes with reasons
          # --junit-xml for test reporting
          # Tests marked with xfail are expected to fail

          echo "::group::Running ecosystem tests with pytest"

          # Run pytest and capture exit code
          pytest ecosystem/scenarios/test_*.py \
            -v \
            --tb=short \
            -ra \
            --junit-xml=test-results.xml \
            --color=yes

          # Capture pytest exit code for potential reporting
          echo "ECOSYSTEM_PYTEST_EXIT_CODE=$?" >> $GITHUB_ENV

          echo "::endgroup::"

          # Parse results for reporting
          if [ -f test-results.xml ]; then
            # Robustly parse JUnit XML using Python
            {
              python3 <<'PYTHON_SCRIPT'
          import xml.etree.ElementTree as ET
          import sys

          try:
              tree = ET.parse("test-results.xml")
              root = tree.getroot()

              # Initialize counters
              total_tests = 0
              total_failures = 0
              total_errors = 0
              total_skipped = 0

              # Handle both single testsuite and testsuites formats
              if root.tag == "testsuites":
                  # Multiple test suites
                  for testsuite in root.findall("testsuite"):
                      total_tests += int(testsuite.get("tests", 0))
                      total_failures += int(testsuite.get("failures", 0))
                      total_errors += int(testsuite.get("errors", 0))
                      total_skipped += int(testsuite.get("skipped", 0))
              elif root.tag == "testsuite":
                  # Single test suite
                  total_tests = int(root.get("tests", 0))
                  total_failures = int(root.get("failures", 0))
                  total_errors = int(root.get("errors", 0))
                  total_skipped = int(root.get("skipped", 0))

              total_passed = total_tests - total_failures - total_errors - total_skipped

              # Output environment variables
              print(f"ECOSYSTEM_TEST_COUNT={total_tests}")
              print(f"ECOSYSTEM_PASSED={total_passed}")
              print(f"ECOSYSTEM_FAILED={total_failures}")
              print(f"ECOSYSTEM_ERRORS={total_errors}")
              print(f"ECOSYSTEM_SKIPPED={total_skipped}")

              # Also output summary for display
              print("-----------------------------------", file=sys.stderr)
              print("Test Results Summary:", file=sys.stderr)
              print(f"  Total tests: {total_tests}", file=sys.stderr)
              print(f"  Passed: {total_passed}", file=sys.stderr)
              print(f"  Failed: {total_failures}", file=sys.stderr)
              print(f"  Errors: {total_errors}", file=sys.stderr)
              print(f"  Skipped: {total_skipped}", file=sys.stderr)
              print("-----------------------------------", file=sys.stderr)

          except Exception as e:
              print(f"Error parsing test results: {e}", file=sys.stderr)
              # Output defaults on error
              print("ECOSYSTEM_TEST_COUNT=0")
              print("ECOSYSTEM_PASSED=0")
              print("ECOSYSTEM_FAILED=0")
              print("ECOSYSTEM_ERRORS=0")
              print("ECOSYSTEM_SKIPPED=0")
          PYTHON_SCRIPT
            } >> $GITHUB_ENV
          else
            # No test results file, save defaults
            echo "-----------------------------------"
            echo "Test Results Summary:"
            echo "  No test results file found"
            echo "-----------------------------------"
            {
              echo "ECOSYSTEM_TEST_COUNT=0"
              echo "ECOSYSTEM_PASSED=0"
              echo "ECOSYSTEM_FAILED=0"
              echo "ECOSYSTEM_ERRORS=0"
              echo "ECOSYSTEM_SKIPPED=0"
            } >> $GITHUB_ENV
          fi

      - name: Install Bencher CLI
        uses: bencherdev/bencher@e2c5af8bc599561e0fb49e933971d3daca3d1230 # main

      - name: Run ecosystem benchmarks with Bencher
        id: bench
        env:
          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
          BENCHER_PROJECT: cribo
          BENCHER_TESTBED: ubuntu-latest
        run: |
          # Prefer the prebuilt binary for the runner
          export CARGO_BIN_EXE_cribo="$PWD/target/release/cribo"

          set -e  # Exit on error

          # Run the bundling benchmark script - outputs BMF JSON
          python3 ecosystem/benchmark_bundling.py > benchmark_output.json

          # Verify the script succeeded and produced valid output
          if [ ! -s benchmark_output.json ]; then
            echo "Error: benchmark_output.json is empty or missing"
            exit 1
          fi

          # Display the metrics for debugging
          echo "📊 Benchmark metrics generated:"
          cat benchmark_output.json

          # Determine branch and comparison settings
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            BRANCH_NAME="pr-${{ github.event.pull_request.number }}"
            START_POINT_FLAGS=(--start-point main --start-point-reset)
            CI_FLAGS=(--ci-only-on-alert --ci-id "${{ github.event.number }}" --ci-number "${{ github.event.number }}")
          else
            BRANCH_NAME="main"
            START_POINT_FLAGS=()
            CI_FLAGS=()
          fi

          # Submit metrics to Bencher with proper branch handling and thresholds
          # Use --format json to capture report UUID for direct linking
          # Capture JSON output while allowing diagnostics to stderr
          bencher run \
            --project "$BENCHER_PROJECT" \
            --token "$BENCHER_API_TOKEN" \
            --branch "$BRANCH_NAME" \
            "${START_POINT_FLAGS[@]}" \
            --testbed "$BENCHER_TESTBED" \
            --adapter json \
            --file benchmark_output.json \
            --github-actions "${{ secrets.GITHUB_TOKEN }}" \
            "${CI_FLAGS[@]}" \
            --threshold-measure bundle_time \
            --threshold-test t_test \
            --threshold-min-sample-size 5 \
            --threshold-max-sample-size 64 \
            --threshold-upper-boundary 0.95 \
            --threshold-measure bundle_size \
            --threshold-test t_test \
            --threshold-min-sample-size 5 \
            --threshold-max-sample-size 64 \
            --threshold-upper-boundary 0.95 \
            --format json 2>&1 | tee bencher_output.txt

          # Extract report UUID from bencher output
          # The output contains benchmark metrics JSON followed by the report JSON
          # We need to extract the last complete JSON object which is the report
          # Use grep to find the line with standalone { that starts the report JSON
          # then tail to get everything from that point, and jq to parse
          REPORT_UUID=$(grep -n '^{$' bencher_output.txt | tail -1 | cut -d: -f1 | xargs -I {} tail -n +{} bencher_output.txt | jq -r '.uuid' 2>/dev/null || echo "")

          if [ -n "$REPORT_UUID" ]; then
            echo "BENCHER_REPORT_UUID=$REPORT_UUID" >> "$GITHUB_ENV"
            echo "📊 Report UUID: $REPORT_UUID"
            echo "📊 Direct link: https://bencher.dev/console/projects/cribo/reports/$REPORT_UUID"
          else
            echo "⚠️ Could not extract report UUID from Bencher output"
            echo "Debug: Checking for JSON start markers in output..."
            grep -n '^{$' bencher_output.txt | head -3
          fi

      - name: Generate test summary report
        if: always()
        id: test-report
        run: |
          # Create a markdown report using grouped commands to avoid shellcheck SC2129
          {
            echo "## 📊 Ecosystem Test Results"
            echo ""
            echo "### 📋 Test Status"
            echo ""

            if [ -n "$ECOSYSTEM_TEST_COUNT" ] && [ "$ECOSYSTEM_TEST_COUNT" -gt 0 ]; then
              echo "**Test Summary:**"
              echo "- Total: $ECOSYSTEM_TEST_COUNT"
              echo "- ✅ Passed: $ECOSYSTEM_PASSED"
              echo "- ❌ Failed: $ECOSYSTEM_FAILED"
              echo "- ⚠️ Errors: $ECOSYSTEM_ERRORS"
              echo "- ⏭️ Skipped: $ECOSYSTEM_SKIPPED"
              echo ""

              if [ "$ECOSYSTEM_FAILED" -gt 0 ] || [ "$ECOSYSTEM_ERRORS" -gt 0 ]; then
                echo "> Note: Tests marked with \`@pytest.mark.xfail\` are expected to fail while issues are being resolved."
                echo ""
              fi
            else
              echo "No ecosystem tests were run."
              echo ""
            fi

            echo "### 📈 Benchmark Results"
            echo ""

            # Add direct report link if available
            if [ -n "$BENCHER_REPORT_UUID" ]; then
              echo "📊 [View detailed benchmark report](https://bencher.dev/console/projects/cribo/reports/$BENCHER_REPORT_UUID)"
              echo ""
            fi

            # Parse and display benchmark results in a collapsible table
            if [ -f bencher_output.txt ]; then
              # Extract the last JSON object (the report) and parse results
              BENCHER_RESULTS=$(grep -n '^{$' bencher_output.txt | tail -1 | cut -d: -f1 | xargs -I {} tail -n +{} bencher_output.txt | jq -r '
                .results[0] // [] |
                map({
                  benchmark: .benchmark.name,
                  bundle_time: (.measures[] | select(.measure.name == "bundle_time") | "\(.metric.value | tonumber | . * 100 | round / 100) \(.measure.units)"),
                  bundle_size: (.measures[] | select(.measure.name == "bundle_size") | "\(.metric.value | tonumber | . * 100 | round / 100) \(.measure.units)")
                }) |
                sort_by(.benchmark) |
                .[] |
                "| \(.benchmark) | \(.bundle_time) | \(.bundle_size) |"
              ' 2>/dev/null || echo "")

              if [ -n "$BENCHER_RESULTS" ]; then
                echo "<details>"
                echo "<summary>📦 Package Bundling Metrics</summary>"
                echo ""
                echo "| Package | Bundle Time | Bundle Size |"
                echo "|---------|-------------|-------------|"
                echo "$BENCHER_RESULTS"
                echo ""
                echo "</details>"
                echo ""
              fi
            fi

            echo "Benchmark metrics are tracked via [Bencher.dev](https://bencher.dev/console/projects/cribo/perf)"
            echo ""
            echo "📊 View detailed performance trends and comparisons on the Bencher dashboard."
          } > ecosystem-report.md

          # Also publish to GitHub Actions Summary tab
          cat ecosystem-report.md >> "$GITHUB_STEP_SUMMARY"

      - name: Post PR comment
        if: github.event_name == 'pull_request'
        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
        with:
          script: |
            const fs = require('fs');
            const report = fs.readFileSync('ecosystem-report.md', 'utf8');

            // Find existing comment
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });

            const botComment = comments.find(comment =>
              comment.user.type === 'Bot' &&
              comment.body.includes('📊 Ecosystem Test Results')
            );

            const body = report + '\n\n<sub>Generated by ecosystem-tests workflow</sub>';

            if (botComment) {
              // Update existing comment
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: botComment.id,
                body: body
              });
            } else {
              // Create new comment
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: body
              });
            }

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: ecosystem-test-results
          retention-days: 14
          if-no-files-found: ignore
          path: |
            test-results.xml
            ecosystem-report.md
            benchmark_output.json