gomplate/.github/workflows/benchmark.yml at main · flanksource/gomplate · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
on:
  pull_request:

name: Benchmark

permissions:
  contents: read
  issues: write
  pull-requests: write

jobs:
  bench:
    name: Benchmark
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Install Go
        uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
        with:
          go-version: 1.26.x
          cache: false
      - name: Checkout code
        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          fetch-depth: 0
      - name: Post "benchmark running" comment
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        with:
          script: |
            const marker = '<!-- benchstat-report -->';
            const body = [
              marker,
              'Benchmark running.',
              '',
              `Base: \`${{ github.event.pull_request.base.sha }}\``,
              `Head: \`${{ github.event.pull_request.head.sha }}\``,
              '',
              'This comment will be updated when benchmarks complete.',
            ].join('\n');

            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              per_page: 100,
            });

            const existing = comments.find(c => (c.body || '').includes(marker));

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body,
              });
            }
      - name: Install benchstat
        run: |
          GOBIN="$PWD/.bin" go install golang.org/x/perf/cmd/benchstat@82a0b07e230d76fa1b3036c383d7a98172f87334
          echo "$PWD/.bin" >> "$GITHUB_PATH"
      - name: Test benchmark summary
        run: |
          python3 -m pip install pytest==9.0.3
          python3 -m pytest .github/scripts/benchstat-summary_test.py
      - name: Prepare base worktree
        run: |
          mkdir -p .tmp/bench
          git worktree add .tmp/bench/base "${{ github.event.pull_request.base.sha }}"
          # Override the base's bench files with HEAD's so both runs execute
          # the same benchmark suite — only the implementation under test differs.
          cp serialize_bench_test.go .tmp/bench/base/serialize_bench_test.go
          cp run_expression_bench_test.go .tmp/bench/base/run_expression_bench_test.go
      - name: Compile benchmark binaries
        run: |
          cd .tmp/bench/base
          go test -c -o "$GITHUB_WORKSPACE/.tmp/bench/base.test" github.com/flanksource/gomplate/v3
          cd "$GITHUB_WORKSPACE"
          go test -c -o .tmp/bench/head.test github.com/flanksource/gomplate/v3
      - name: Benchmark interleaved base and head
        run: |
          : > bench-base.txt
          : > bench-head.txt
          for sample in {1..10}
          do
            echo "Running base sample ${sample}/10"
            .tmp/bench/base.test -test.run=^$ -test.bench='Benchmark(Serialize|RunExpressionContext|CELEnvExtend|CELProgramEvaluation|RunExpressionNativeInput)' -test.benchmem=true -test.benchtime=1s -test.count=1 -test.timeout=20m >> bench-base.txt
            echo "Running head sample ${sample}/10"
            .tmp/bench/head.test -test.run=^$ -test.bench='Benchmark(Serialize|RunExpressionContext|CELEnvExtend|CELProgramEvaluation|RunExpressionNativeInput)' -test.benchmem=true -test.benchtime=1s -test.count=1 -test.timeout=20m >> bench-head.txt
          done
      - name: Compare
        run: |
          benchstat bench-base.txt bench-head.txt > benchstat.txt
          benchstat -format csv bench-base.txt bench-head.txt > benchstat.csv
          python3 .github/scripts/benchstat-summary.py --no-fail benchstat.csv > bench-summary.md

          {
            echo "<!-- benchstat-report -->"
            echo "## Benchstat"
            echo ""
            echo "Base: \`${{ github.event.pull_request.base.sha }}\`"
            echo "Head: \`${{ github.event.pull_request.head.sha }}\`"
            echo ""
            while IFS= read -r line
            do
              echo "$line"
            done < bench-summary.md
            echo ""
            echo '<details>'
            echo '<summary>Full benchstat output</summary>'
            echo ""
            echo '```text'
            while IFS= read -r line
            do
              echo "$line"
            done < benchstat.txt
            echo '```'
            echo ""
            echo '</details>'
          } > bench-report.md
      - name: Upload artifacts
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: benchmark-results
          path: |
            bench-base.txt
            bench-head.txt
            benchstat.txt
            benchstat.csv
            bench-summary.md
            bench-report.md
          retention-days: 14
      - name: Post report to PR
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
        with:
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('bench-report.md', 'utf8');
            const marker = '<!-- benchstat-report -->';

            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              per_page: 100,
            });

            const existing = comments.find(c => (c.body || '').includes(marker));

            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body: body
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body: body
              });
            }
      - name: Check for regressions
        run: python3 .github/scripts/benchstat-summary.py --threshold 5 benchstat.csv > /dev/null