Skip to content

Commit 87fb095

Browse files
ileitchileitch
committed
CI benchmarks
1 parent e683568 commit 87fb095

1 file changed

Lines changed: 122 additions & 0 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
name: Benchmark
2+
on:
3+
issue_comment:
4+
types: [created]
5+
pull_request:
6+
types: [labeled]
7+
env:
8+
MISE_EXPERIMENTAL: 1
9+
10+
jobs:
11+
benchmark:
12+
name: Benchmark
13+
if: |
14+
(github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/benchmark')) ||
15+
(github.event_name == 'pull_request' && github.event.label.name == 'benchmark')
16+
runs-on: macos-26
17+
permissions:
18+
contents: read
19+
pull-requests: write
20+
steps:
21+
- name: Get PR number
22+
id: pr-number
23+
run: |
24+
if [ "${{ github.event_name }}" = "issue_comment" ]; then
25+
echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
26+
else
27+
echo "number=${{ github.event.pull_request.number }}" >> "$GITHUB_OUTPUT"
28+
fi
29+
30+
- name: Get PR details
31+
id: pr
32+
env:
33+
GH_TOKEN: ${{ github.token }}
34+
run: |
35+
PR_DATA=$(gh api repos/${{ github.repository }}/pulls/${{ steps.pr-number.outputs.number }})
36+
HEAD_SHA=$(echo "$PR_DATA" | jq -r '.head.sha')
37+
HEAD_REF=$(echo "$PR_DATA" | jq -r '.head.ref')
38+
BASE_REF=$(echo "$PR_DATA" | jq -r '.base.ref')
39+
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
40+
echo "head_ref=$HEAD_REF" >> "$GITHUB_OUTPUT"
41+
echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT"
42+
43+
- name: Add reaction to comment
44+
if: github.event_name == 'issue_comment'
45+
env:
46+
GH_TOKEN: ${{ github.token }}
47+
run: |
48+
gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions \
49+
-f content='+1'
50+
51+
- uses: actions/checkout@master
52+
with:
53+
ref: ${{ steps.pr.outputs.head_sha }}
54+
fetch-depth: 0
55+
56+
- name: Select Xcode version
57+
run: sudo xcode-select -s /Applications/Xcode_26.2.0.app
58+
59+
- uses: jdx/mise-action@v2
60+
61+
- name: Install hyperfine
62+
run: brew install hyperfine
63+
64+
- name: Get merge-base
65+
id: commits
66+
run: |
67+
git fetch origin ${{ steps.pr.outputs.base_ref }}
68+
MERGE_BASE=$(git merge-base HEAD origin/${{ steps.pr.outputs.base_ref }})
69+
echo "merge_base=$MERGE_BASE" >> "$GITHUB_OUTPUT"
70+
echo "Head SHA: ${{ steps.pr.outputs.head_sha }}"
71+
echo "Merge-base SHA: $MERGE_BASE"
72+
73+
- name: Build PR
74+
run: mise r build --arch arm64
75+
76+
- name: Benchmark HEAD
77+
run: |
78+
hyperfine --warmup 3 --export-json head-benchmark.json './.build/arm64-apple-macosx/release/periphery scan --quiet --skip-build'
79+
80+
- name: Build merge-base
81+
run: |
82+
git checkout ${{ steps.commits.outputs.merge_base }}
83+
mise r build --arch arm64
84+
85+
- name: Benchmark merge-base
86+
run: |
87+
hyperfine --warmup 3 --export-json base-benchmark.json './.build/arm64-apple-macosx/release/periphery scan --quiet --skip-build'
88+
89+
- name: Post results
90+
env:
91+
GH_TOKEN: ${{ github.token }}
92+
run: |
93+
HEAD_MEAN=$(jq '.results[0].mean' head-benchmark.json)
94+
BASE_MEAN=$(jq '.results[0].mean' base-benchmark.json)
95+
HEAD_STDDEV=$(jq '.results[0].stddev' head-benchmark.json)
96+
BASE_STDDEV=$(jq '.results[0].stddev' base-benchmark.json)
97+
98+
CHANGE=$(echo "scale=2; (($HEAD_MEAN - $BASE_MEAN) / $BASE_MEAN) * 100" | bc)
99+
100+
COMMENT="## Benchmark Results
101+
102+
| Commit | Mean (s) | Std Dev (s) |
103+
|--------|----------|-------------|
104+
| Merge-base (\`$(echo ${{ steps.commits.outputs.merge_base }} | cut -c1-7)\`) | $(printf "%.3f" $BASE_MEAN) | ±$(printf "%.3f" $BASE_STDDEV) |
105+
| HEAD (\`$(echo ${{ steps.pr.outputs.head_sha }} | cut -c1-7)\`) | $(printf "%.3f" $HEAD_MEAN) | ±$(printf "%.3f" $HEAD_STDDEV) |
106+
107+
**Change: $(printf "%+.1f%%" $CHANGE)**"
108+
109+
gh pr comment ${{ steps.pr-number.outputs.number }} --body "$COMMENT"
110+
111+
- name: Remove benchmark label
112+
if: github.event_name == 'pull_request'
113+
env:
114+
GH_TOKEN: ${{ github.token }}
115+
run: |
116+
gh pr edit ${{ steps.pr-number.outputs.number }} --remove-label benchmark
117+
118+
- name: Upload benchmark results
119+
uses: actions/upload-artifact@v4
120+
with:
121+
name: benchmark-results
122+
path: "*-benchmark.json"

0 commit comments

Comments
 (0)