Total what the two runs spent per phase

The report says which models changed phase and which got slower, and
nothing about what the run as a whole cost. Testing pull request 16696
moved 83 of 19842 models and spent 8% more in the backend, which the
report had no way of saying.

The page and the comment now carry a table of both runs summed per
phase over the models they have in common:

    | Phase    | master   | pr/16696 | Change | Change, same phase |
    | -------- | -------- | -------- | ------ | ------------------ |
    | frontend |  1:44:57 |  1:44:37 |  -0.3% |              -0.6% |
    | backend  |  3:12:28 |  3:27:54 |  +8.0% |              +7.6% |

The last column is the same sum over only the models that reached the
same phase in both runs. A model that now fails in the backend stops
paying for the compilation and the simulation it no longer reaches, so
without that column a run that broke models reads as a faster one:
here the whole simulation looks 1.9% cheaper, while the models that
still simulate spend 0.4% more.

`exectime`, the whole run of a model, is the total.

Assisted-by: Claude Opus 5 (1M context)

Author: sjoelund

README.md

@@ -462,6 +462,11 @@ kind of page as the nightly regression reports, next to `00_comment.md`, a
 summary to comment on the pull request with. Both are published with the other
 reports.
 
+Both also total what the two runs spent per phase on the models they have in
+common, next to the same totals over only the models that reached the same
+phase in both: a model that fails earlier stops paying for the phases it no
+longer reaches, which makes those phases look cheaper than they are.
+
 `--comment` posts that summary on the pull request, replacing the one an earlier
 run posted. It posts as whoever the token belongs to: `GITHUB_TOKEN` or
 `GH_TOKEN` in the environment, or the account [`gh`](https://cli.github.com) is

pr-report.py

@@ -40,6 +40,8 @@
 timeAbs = 10     # Ignore performance regressions for times <10s...
 
 PHASES = [(1,"frontend"),(2,"backend"),(3,"simcode"),(4,"templates"),(5,"compile"),(6,"simulate")]
+# The columns the totals are summed over; exectime is the whole run of a model.
+TIMES = ["frontend","backend","simcode","templates","compile","simulate","verify","exectime"]
 
 m = re.match(r"^(?:pr[-/])?([0-9]+)$", args.pullrequest.strip())
 if not m:
@@ -197,6 +199,25 @@ def changedModels(table1, date1, table2, date2, libnames):
   return cursor.fetchall()
 
 
+def phaseTotals(table1, date1, table2, date2, libnames):
+  """What the two runs spent, per phase, on the models they both have.
+
+  Twice: over every compared model, and over those that reached the same phase
+  in both runs. A model that now fails earlier stops paying for the phases it
+  no longer reaches, which makes those phases look cheaper than they are.
+  """
+  inlibs = ",".join("'%s'" % libname for libname in sorted(libnames))
+  cols = ["sum(a.%s),sum(b.%s)" % (t, t) for t in TIMES]
+  cols += ["sum(CASE WHEN a.finalphase=b.finalphase THEN a.%s ELSE 0 END),"
+           "sum(CASE WHEN a.finalphase=b.finalphase THEN b.%s ELSE 0 END)" % (t, t)
+           for t in TIMES]
+  cols += [db.countIf("a.finalphase=b.finalphase")]
+  query = """SELECT %s FROM %s AS a JOIN %s AS b ON a.libname=b.libname AND a.model=b.model
+    WHERE a.date=? AND b.date=? AND a.libname IN (%s) AND a.finalphase>=0 AND b.finalphase>=0
+  """ % (",".join(cols), db.quote(table1), db.quote(table2), inlibs)
+  return [v or 0 for v in cursor.execute(query, (date1, date2)).fetchone()]
+
+
 prdate = newestRun(branch, args.date)
 if not prdate:
   raise SystemExit("No results for %s%s" % (branch, " at or before %d" % args.date if args.date else ""))
@@ -223,8 +244,10 @@ def changedModels(table1, date1, table2, date2, libnames):
   groups.setdefault((d1, d2), []).append(libname)
 
 changes = []
+totals = [0] * (4*len(TIMES) + 1)
 for ((d1, d2), libs) in sorted(groups.items()):
   changes += changedModels(baseline, d1, branch, d2, libs)
+  totals = [t + v for (t, v) in zip(totals, phaseTotals(baseline, d1, branch, d2, libs))]
 changes = sorted(changes, key=lambda x: (x[1], x[0]))
 
 # Models one of the runs has and the other does not: a library that grew a model,
@@ -328,6 +351,24 @@ def classify(group, times):
         "difference can also come from something merged into %s since the pull request was "
         "branched." % (baseline, baseline))
 
+def relative(before, after):
+  return "%+.1f%%" % (100.0*(after-before)/before) if before else ""
+
+numSamePhase = totals[-1]
+totalrows = []
+markdowntotals = []
+for (i, name) in enumerate(TIMES):
+  (t1, t2) = totals[2*i:2*i+2]
+  (s1, s2) = totals[2*len(TIMES)+2*i:2*len(TIMES)+2*i+2]
+  cells = ["total" if name == "exectime" else name,
+           friendlyStr(t1), friendlyStr(t2), relative(t1, t2), relative(s1, s2)]
+  totalrows.append("<tr>%s</tr>" % "".join("<td>%s</td>" % c for c in cells))
+  markdowntotals.append("| %s |" % " | ".join(cells))
+totalnote = ("Time spent on the %d models both runs have. The last column counts only the %d "
+             "that reached the same phase in both: a model that fails earlier stops paying "
+             "for the phases it no longer reaches."
+             % (numCompared, numSamePhase))
+
 reportname = "%s..%s.html" % (dateStr(basedate), dateStr(prdate))
 historydir = os.path.join(args.historypath, branch)
 os.makedirs(historydir, exist_ok=True)
@@ -347,6 +388,8 @@ def classify(group, times):
   ("#HOST1#", html.escape(", ".join(sorted(basehosts)))),
   ("#HOST2#", html.escape(", ".join(sorted(prhosts)))),
   ("#NUMCOMPARED#", str(numCompared)),
+  ("#TOTALS#", "\n".join(totalrows)),
+  ("#TOTALNOTE#", totalnote),
   ("#NUMIMPROVE#", str(counts["improved"])),
   ("#NUMREGRESSION#", str(counts["regression"])),
   ("#NUMPERFIMPROVE#", str(counts["performance improved"])),
@@ -389,6 +432,10 @@ def classify(group, times):
             "%d models compared, **%d improved, %d regressions**, performance %d improved, %d regressions."
             % (numCompared, counts["improved"], counts["regression"],
                counts["performance improved"], counts["performance regression"]),
+            "",
+            "| Phase | `%s` | `%s` | Change | Change, same phase |" % (baseline, branch),
+            "| --- | --- | --- | --- | --- |"] + markdowntotals + [
+            "", totalnote,
             "", "[Full report](%s)" % reporturl, ""]
 if markdownrows:
   markdown += ["<details><summary>%d models affected</summary>" % len(markdownrows), "",

pr.html.tpl

@@ -42,6 +42,15 @@ the pull request was branched.</p>
 <tr><td>Models only in the baseline run</td><td>#NUMONLYBASELINE#</td></tr>
 </table>
 
+<h2>Time</h2>
+
+<p>#TOTALNOTE#</p>
+
+<table>
+<tr><th>Phase</th><th>#BASELINE#</th><th>#BRANCH#</th><th>Change</th><th>Change, same phase</th></tr>
+#TOTALS#
+</table>
+
 <h2>Library Changes</h2>
 <table>
 <tr><th>Library</th><th>Change</th></tr>