Skip to content

Commit 06c3bc9

Browse files
JuanVqzJuanVqz
authored
Merge pull request #252 from fastruby/feature/results-site-view-2
Show every Ruby's speed side by side on the results site
2 parents 96cd831 + 491f640 commit 06c3bc9

9 files changed

Lines changed: 835 additions & 197 deletions

File tree

‎.github/workflows/benchmarks.yml‎

Lines changed: 7 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
name: Benchmarks
22

3+
# Checks that every benchmark runs on every Ruby (PRs: only the files they change).
4+
# It publishes nothing: the results site comes from results-site.yml.
35
# Always runs, so the benchmarks-ok check at the end always reports and can be
46
# required. The changes job decides whether there is anything to benchmark.
57
on:
@@ -67,14 +69,12 @@ jobs:
6769
- { ruby: ruby_4.0, variant: zjit, flags: --zjit }
6870
- { ruby: ruby_head, variant: zjit, flags: --zjit }
6971

70-
# Read by docker/run-benchmarks.sh and docker/collect_results.rb.
7172
env:
73+
# Read by docker/run-benchmarks.sh.
7274
RUBY_VARIANT: ${{ matrix.variant }}
7375
RUBY_VARIANT_FLAGS: ${{ matrix.flags }}
74-
RESULTS_DIR: results
75-
RESULTS_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
76-
RESULTS_PR: ${{ github.event.pull_request.number }}
77-
RESULTS_LABEL: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }}
76+
# The build's name in the step names and the failure report, for example ruby_3.4+yjit.
77+
BUILD: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }}
7878

7979
steps:
8080
# Plain Rubies only: shared reports do not record the Ruby or its flags,
@@ -84,113 +84,13 @@ jobs:
8484
run: |
8585
echo "SHARE=1" >> "$GITHUB_ENV"
8686
- uses: actions/checkout@v4
87-
- name: Run benchmarks on ${{ env.RESULTS_LABEL }}
87+
- name: Run benchmarks on ${{ env.BUILD }}
8888
env:
8989
FILES: ${{ needs.changes.outputs.files }}
9090
run: .github/scripts/run-benchmarks.sh ${{ matrix.ruby }}
9191
- name: Report failed benchmarks
9292
if: failure()
93-
run: .github/scripts/report-failures.sh benchmarks.log ${{ env.RESULTS_LABEL }}
94-
- name: Upload results
95-
uses: actions/upload-artifact@v4
96-
if: always()
97-
with:
98-
name: results-${{ env.RESULTS_LABEL }}
99-
path: results/
100-
if-no-files-found: warn
101-
102-
# Builds the results site from every job's results, also when some jobs
103-
# failed, so a PR gets a preview (the site-preview artifact, named outside
104-
# the results-* pattern so a re-run never downloads it). Skipped when the
105-
# benchmark jobs did not run (a lint failure). Only a push to
106-
# main where every job passed publishes it, so a partial run never replaces
107-
# the live site. Not required: a Pages problem never blocks a merge.
108-
site:
109-
needs: [changes, rake]
110-
if: ${{ !cancelled() && needs.changes.outputs.run == 'true' && needs.rake.result != 'skipped' }}
111-
runs-on: ubuntu-latest
112-
113-
steps:
114-
- uses: actions/checkout@v4
115-
- uses: actions/download-artifact@v4
116-
with:
117-
pattern: results-*
118-
path: results/
119-
merge-multiple: true
120-
- name: Build the results site
121-
# A failed benchmark job leaves no result file, like a benchmark that needs a newer Ruby, so the page says missing results may be crashes.
122-
env:
123-
RESULTS_INCOMPLETE: ${{ needs.rake.result != 'success' && '1' || '' }}
124-
run: docker compose run --rm -T -e RESULTS_INCOMPLETE --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site
125-
- name: Upload the site preview
126-
id: preview
127-
uses: actions/upload-artifact@v4
128-
with:
129-
name: site-preview
130-
path: _site/
131-
# A re-run replaces the earlier attempt's preview.
132-
overwrite: true
133-
# A one-click link on the run's Summary page; it needs no extra permissions, so it works for PRs from forks too.
134-
- name: Link the site preview in the run summary
135-
env:
136-
PREVIEW_URL: ${{ steps.preview.outputs.artifact-url }}
137-
run: |
138-
{
139-
echo "### Results site preview"
140-
echo ""
141-
echo "[Download the site preview]($PREVIEW_URL) (a zip, needs a GitHub login), unzip it and open \`index.html\` in a browser."
142-
} >> "$GITHUB_STEP_SUMMARY"
143-
- name: Upload the site for GitHub Pages
144-
if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success'
145-
uses: actions/upload-pages-artifact@v5
146-
with:
147-
path: _site/
148-
149-
deploy:
150-
needs: [rake, site]
151-
if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success' && needs.site.result == 'success'
152-
runs-on: ubuntu-latest
153-
# Only this job can publish; the rest of the workflow keeps the default
154-
# token permissions.
155-
permissions:
156-
pages: write
157-
id-token: write
158-
environment:
159-
name: github-pages
160-
url: ${{ steps.deployment.outputs.page_url }}
161-
# One deploy at a time; a newer merge waits instead of cancelling one that
162-
# is halfway through.
163-
concurrency:
164-
group: pages
165-
cancel-in-progress: false
166-
167-
steps:
168-
# Runs can finish out of order, so an older run must not put its results back over a newer one.
169-
# When main moved on, ask pick-benchmarks.sh (the same rule CI uses) whether the newer commits run benchmarks.
170-
# If they do, their own run deploys newer results; if not (a README-only merge), this run's results are still the newest.
171-
- uses: actions/checkout@v4
172-
with:
173-
ref: main
174-
fetch-depth: 0
175-
- name: Check no newer run on main will deploy
176-
id: newest
177-
run: |
178-
if [ "$(git rev-parse HEAD)" = "$GITHUB_SHA" ]; then
179-
echo "deploy=true" >> "$GITHUB_OUTPUT"
180-
exit 0
181-
fi
182-
GITHUB_EVENT_NAME=push GITHUB_OUTPUT=newer.txt .github/scripts/pick-benchmarks.sh "$GITHUB_SHA"
183-
if grep -q '^run=true' newer.txt; then
184-
echo "main moved on and its newer commits run benchmarks, so their run deploys; skipping."
185-
echo "deploy=false" >> "$GITHUB_OUTPUT"
186-
else
187-
echo "main moved on, but nothing since this run affects benchmarks; deploying."
188-
echo "deploy=true" >> "$GITHUB_OUTPUT"
189-
fi
190-
- name: Deploy to GitHub Pages
191-
id: deployment
192-
if: steps.newest.outputs.deploy == 'true'
193-
uses: actions/deploy-pages@v5
93+
run: .github/scripts/report-failures.sh benchmarks.log ${{ env.BUILD }}
19494

19595
# The check to require on main. Passes when every benchmark job passed, or
19696
# when there was nothing to benchmark.

‎.github/workflows/results-site.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Results site
2+
3+
# Builds and publishes the results site (https://fastruby.github.io/fast-ruby/).
4+
# Every Ruby runs each benchmark on the same machine, so the site can compare i/s across Rubies; benchmarks.yml runs each Ruby on its own machine and only checks that the benchmarks run.
5+
# Weekly, because a full run takes about 3 hours per shard; run it by hand to publish sooner.
6+
on:
7+
schedule:
8+
# Sunday 03:00 UTC, away from the working day's PR runs.
9+
- cron: '0 3 * * 0'
10+
workflow_dispatch:
11+
12+
jobs:
13+
# 6 machines, each running every 6th benchmark file on all 26 builds, with the newest released Ruby run again between them (script/run_cross_ruby.rb).
14+
shard:
15+
name: shard ${{ matrix.shard }}
16+
runs-on: ubuntu-latest
17+
# 35 passes of about 13 files each; the job limit is 6 hours.
18+
timeout-minutes: 330
19+
20+
strategy:
21+
fail-fast: false
22+
matrix:
23+
shard: [0, 1, 2, 3, 4, 5]
24+
25+
# Read by docker/collect_results.rb.
26+
env:
27+
RESULTS_COMMIT: ${{ github.sha }}
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
- name: Run every build on this machine
32+
run: ruby script/run_cross_ruby.rb --shard ${{ matrix.shard }} --shards 6 --fresh-images --out results
33+
# Also when a pass failed: the shard's other results are kept, to see what went wrong (the site is not built from an incomplete run).
34+
- name: Upload results
35+
uses: actions/upload-artifact@v4
36+
if: always()
37+
with:
38+
name: results-shard-${{ matrix.shard }}
39+
path: results/
40+
retention-days: 90
41+
# A re-run of a failed shard replaces its earlier attempt.
42+
overwrite: true
43+
44+
# Builds both views from all six shards and publishes them, only for a complete run on main.
45+
site:
46+
needs: shard
47+
if: github.ref_name == 'main'
48+
runs-on: ubuntu-latest
49+
permissions:
50+
pages: write
51+
id-token: write
52+
# Reads this workflow's earlier runs (the newest-results check below).
53+
actions: read
54+
environment:
55+
name: github-pages
56+
url: ${{ steps.deployment.outputs.page_url }}
57+
# One deploy at a time; a newer run waits instead of cancelling one that is halfway through.
58+
concurrency:
59+
group: pages
60+
cancel-in-progress: false
61+
62+
steps:
63+
- uses: actions/checkout@v4
64+
# Never put older results back: a re-run of an old run, or two runs finishing out of order, must not publish over a newer run's results.
65+
# Run ids only grow and a re-run keeps its id, so a newer run has a higher id (and, since main cannot be force-pushed, a commit at least as new).
66+
- name: Check no newer results are live
67+
id: newest
68+
env:
69+
GH_TOKEN: ${{ github.token }}
70+
run: |
71+
# The highest id among recent successful runs, so the order the list comes back in does not matter.
72+
live=$(gh run list --workflow results-site.yml --branch main --status success --limit 20 --json databaseId --jq 'map(.databaseId) | max // empty')
73+
if [ -n "$live" ] && [ "$live" -gt "$GITHUB_RUN_ID" ]; then
74+
echo "The live site has results from run $live, newer than this run ($GITHUB_RUN_ID); not publishing."
75+
echo "publish=false" >> "$GITHUB_OUTPUT"
76+
else
77+
echo "publish=true" >> "$GITHUB_OUTPUT"
78+
fi
79+
- uses: actions/download-artifact@v4
80+
if: steps.newest.outputs.publish == 'true'
81+
with:
82+
pattern: results-shard-*
83+
path: results/
84+
merge-multiple: true
85+
- name: Build the results site
86+
if: steps.newest.outputs.publish == 'true'
87+
run: docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site
88+
- name: Upload the site for GitHub Pages
89+
if: steps.newest.outputs.publish == 'true'
90+
uses: actions/upload-pages-artifact@v5
91+
with:
92+
path: _site/
93+
- name: Deploy to GitHub Pages
94+
id: deployment
95+
if: steps.newest.outputs.publish == 'true'
96+
uses: actions/deploy-pages@v5

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
/Gemfile.lock
44
/results/
55
/_site/
6+
/cross-ruby/

‎CONTRIBUTING.md‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ These idioms list here are trying to satisfy following goals:
1212

1313
- [Note on entry](#note-on-entry)
1414
- [Running it on other Rubies](#running-it-on-other-rubies)
15+
- [The results site](#the-results-site)
1516
- [Benchmarks that need a newer Ruby](#benchmarks-that-need-a-newer-ruby)
1617
- [License](#license)
1718

@@ -104,23 +105,25 @@ RUBY_VARIANT=yjit RUBY_VARIANT_FLAGS=--yjit docker compose run --rm ruby_3.4 cod
104105
RUBY_VARIANT=zjit RUBY_VARIANT_FLAGS=--zjit docker compose run --rm ruby_4.0 code/your-new/entry.rb
105106
```
106107

107-
To keep the results, set `RESULTS_DIR`. Each benchmark then also writes its
108-
report as JSON to `results/<label>/`, with the Ruby, its flags and the machine
109-
it ran on:
108+
## The results site
109+
110+
The results site (https://fastruby.github.io/fast-ruby/) compares Rubies, so every build of a benchmark has to run on the same machine.
111+
`script/run_cross_ruby.rb` (Ruby 3 on your machine, it calls Docker) does that, and runs the newest released MRI again after every 3 builds, so the site can show how steady the machine was.
112+
To see a few benchmarks the way the site shows them, run them on a few builds, build the site into `_site/`, then open `_site/index.html` in a browser:
110113

111114
```
112-
RESULTS_DIR=results RESULTS_LABEL=ruby_3.4 docker compose run --rm ruby_3.4 code/your-new/entry.rb
115+
ruby script/run_cross_ruby.rb --files code/date/iso8601-vs-parse.rb,code/string/gsub-vs-tr.rb --builds ruby_3.4,ruby_3.4+yjit,ruby_4.0 --out cross-ruby
116+
docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb cross-ruby _site
113117
```
114118

115-
To see those results the way the results site shows them, build the site
116-
into `_site/`, then open `_site/index.html` in a browser:
119+
`ruby script/run_cross_ruby.rb --help` lists the options.
120+
The builds are the ones in the CI matrix (`.github/workflows/benchmarks.yml`), so a new Ruby added there and in `compose.yaml` is measured too, and once released it becomes both the reference and the Ruby the site opens on.
121+
On an Apple silicon Mac, leave out `ruby_2.1` and `jruby_9.1` (they run under emulation, so their numbers are not comparable) and `ruby_3.1+yjit` (Ruby 3.1's YJIT only exists on x86-64).
117122

118-
```
119-
docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site
120-
```
123+
CI has two workflows:
121124

122-
CI does the same after every run: the site is attached to the run as the
123-
`site-preview` artifact, and published to GitHub Pages from `main`.
125+
- `.github/workflows/benchmarks.yml` checks that every benchmark runs on every Ruby: on a PR, the benchmark files it changes; on `main`, all of them when a benchmark or a shared file changed. It publishes nothing.
126+
- `.github/workflows/results-site.yml` runs every build on every file once a week, split over 6 machines, and publishes the site. Run it by hand from the Actions tab to publish sooner.
124127

125128
## Benchmarks that need a newer Ruby
126129

‎compose.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,11 @@ x-benchmark: &benchmark
2020
- RESULTS_DIR
2121
- RESULTS_LABEL
2222
- RESULTS_COMMIT
23-
- RESULTS_PR
23+
# Cross-Ruby runs only (script/run_cross_ruby.rb).
24+
- RESULTS_SHARD
25+
- RESULTS_SHARDS
26+
- RESULTS_PASS
27+
- RESULTS_REFERENCE
2428
- GITHUB_RUN_ID
2529
- GITHUB_RUN_ATTEMPT
2630
- RUNNER_NAME

‎docker/collect_results.rb‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,16 @@ def write(report)
4848
"flags" => env("RUBY_VARIANT_FLAGS"),
4949
"jit" => jit,
5050
"commit" => env("RESULTS_COMMIT"),
51-
"pr" => env("RESULTS_PR") && env("RESULTS_PR").to_i,
5251
"run_id" => env("GITHUB_RUN_ID"),
5352
"run_attempt" => env("GITHUB_RUN_ATTEMPT"),
5453
"runner" => env("RUNNER_NAME"),
54+
# Set by script/run_cross_ruby.rb, which runs every build on one machine and the reference build between them; nil otherwise.
55+
"shard" => env("RESULTS_SHARD") && env("RESULTS_SHARD").to_i,
56+
"shards" => env("RESULTS_SHARDS") && env("RESULTS_SHARDS").to_i,
57+
"pass" => env("RESULTS_PASS") && env("RESULTS_PASS").to_i,
58+
"reference" => env("RESULTS_REFERENCE") == "1",
59+
# When this report finished, to place it between the reference passes.
60+
"measured_at" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
5561
"environment" => environment,
5662
"entries" => report.data
5763
}))

0 commit comments

Comments
 (0)