ci: self-hosted reliability — case-opt timeout/clean, monitor crash, compiler labels (#1643)

Author: sbryngelson

.github/scripts/case-optimization-benchmarks.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Single source of truth for the benchmark cases exercised by the
+# case-optimization CI. The pre-build (compiles the case-optimized binaries)
+# and the run (executes them) MUST iterate the same list in the same order:
+# sharding partitions this list by index, so any drift silently pre-builds or
+# runs the wrong cases. Sourced by prebuild-case-optimization.sh and
+# run_case_optimization.sh.
+benchmarks=(
+    benchmarks/5eq_rk3_weno3_hllc/case.py
+    benchmarks/viscous_weno5_sgb_acoustic/case.py
+    benchmarks/hypo_hll/case.py
+    benchmarks/ibm/case.py
+    benchmarks/igr/case.py
+)
+
+# Parse an optional "$job_shard" of the form "i/N" (e.g. "2/3"). On success
+# sets caseopt_shard_idx / caseopt_shard_count (both 1 when unset — a single
+# shard covering every case). Aborts on a malformed value.
+caseopt_parse_shard() {
+    caseopt_shard_idx=1
+    caseopt_shard_count=1
+    [ -z "${job_shard:-}" ] && return 0
+    caseopt_shard_idx="${job_shard%%/*}"
+    caseopt_shard_count="${job_shard##*/}"
+    case "$caseopt_shard_idx"   in ''|*[!0-9]*|0*) echo "ERROR: bad shard '$job_shard' (expected i/N)"; exit 1 ;; esac
+    case "$caseopt_shard_count" in ''|*[!0-9]*|0*) echo "ERROR: bad shard '$job_shard' (expected i/N)"; exit 1 ;; esac
+    if [ "$job_shard" != "$caseopt_shard_idx/$caseopt_shard_count" ] \
+       || [ "$caseopt_shard_idx" -lt 1 ] || [ "$caseopt_shard_idx" -gt "$caseopt_shard_count" ]; then
+        echo "ERROR: bad shard '$job_shard' (expected i/N with 1 <= i <= N)"; exit 1
+    fi
+}
+
+# 0 (true) if the 1-based case index $1 belongs to this shard. Shard i owns
+# every Nth case: indices i, i+N, i+2N, ...
+caseopt_case_in_shard() {
+    [ $((($1 - 1) % caseopt_shard_count)) -eq $((caseopt_shard_idx - 1)) ]
+}

.github/scripts/monitor_slurm_job.sh

@@ -122,62 +122,39 @@ done
 
 echo "=== Streaming output for job $job_id ==="
 
-# Start tail and redirect its output to file descriptor 3 for multiplexing
-# This allows us to stream tail output while also printing heartbeat messages
-exec 3< <(stdbuf -oL -eL tail -f "$output_file" 2>&1)
+# Stream the job's output to the step log with a plain backgrounded `tail`.
+# This previously used a timed read (`read -t 1`) over a `tail -f` process
+# substitution; when that pipe broke, bash could crash in the read-builtin's
+# alarm/longjmp unwind path (SIGSEGV), and the EXIT trap would then cancel a
+# still-healthy job. A backgrounded tail avoids that construct entirely, and
+# the final `cat` below reprints the whole file so nothing is lost if tail is
+# killed mid-flush.
+stdbuf -oL -eL tail -f "$output_file" 2>&1 &
 tail_pid=$!
 
-# Monitor job status and stream output simultaneously
+# Poll job status until it reaches a terminal state; streaming happens
+# independently in the background tail above.
 last_heartbeat=$(date +%s)
-
 while true; do
-  # Try to read from tail output (non-blocking via timeout)
-  # Read multiple lines if available to avoid falling behind
-  lines_read=0
-  while IFS= read -r -t 1 line <&3 2>/dev/null; do
-    echo "$line"
-    lines_read=$((lines_read + 1))
-    last_heartbeat=$(date +%s)
-    # Limit burst reads to avoid starving the status check
-    if [ $lines_read -ge 100 ]; then
-      break
-    fi
-  done
-
-  # Check job status
-  current_time=$(date +%s)
   state=$(get_job_state "$job_id")
 
   if is_terminal_state "$state"; then
     echo "[$(date +%H:%M:%S)] Job $job_id reached terminal state: $state"
     break
-  else
-    # Print heartbeat if no output for 60 seconds
-    if [ $((current_time - last_heartbeat)) -ge 60 ]; then
-      echo "[$(date +%H:%M:%S)] Job $job_id state=$state (no new output for 60s)..."
-      last_heartbeat=$current_time
-    fi
   fi
 
-  # Sleep briefly between status checks
-  sleep 1
-done
-
-# Drain any remaining output from tail after job completes
-echo "Draining remaining output..."
-drain_count=0
-while IFS= read -r -t 1 line <&3 2>/dev/null; do
-  echo "$line"
-  drain_count=$((drain_count + 1))
-  # Safety limit to avoid infinite loop
-  if [ $drain_count -ge 10000 ]; then
-    echo "Warning: Truncating remaining output after 10000 lines"
-    break
+  # Periodic heartbeat so the CI log never looks stalled during quiet phases.
+  current_time=$(date +%s)
+  if [ $((current_time - last_heartbeat)) -ge 60 ]; then
+    echo "[$(date +%H:%M:%S)] Job $job_id state=$state..."
+    last_heartbeat=$current_time
   fi
+
+  sleep 5
 done
 
-# Close the file descriptor and kill tail
-exec 3<&-
+# Give tail a moment to flush the final lines, then stop streaming.
+sleep 2
 kill "${tail_pid}" 2>/dev/null || true
 tail_pid=""
 

.github/scripts/prebuild-case-optimization.sh

@@ -22,34 +22,16 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
-# Optional sharding (format "i/N", e.g. "1/2"), set by submit-slurm-job.sh's
-# [shard] argument via $job_shard: shard i builds every Nth case of the sorted
-# case list. Unset = build all cases in one job (default; other clusters).
+# Benchmark list + optional sharding ("i/N", e.g. "1/3", set by
+# submit-slurm-job.sh's [shard] argument via $job_shard): shard i builds every
+# Nth case of the shared list. Unset = build all cases in one job (default;
+# other clusters). The list is the single source of truth shared with
+# run_case_optimization.sh so pre-built binaries and run cases never drift.
+source .github/scripts/case-optimization-benchmarks.sh
+caseopt_parse_shard
 shard="${job_shard:-}"
-if [ -n "$shard" ]; then
-    # Validate full shape: must be exactly "digits/digits" — one slash with
-    # non-empty, purely numeric, non-leading-zero parts on both sides.
-    # Split first, then validate each part independently so that inputs like
-    # "1/" "/2" "//" "1/2/3" "a/b" "12" are all caught before any arithmetic.
-    shard_idx="${shard%%/*}"
-    shard_count="${shard##*/}"
-    # Reject if no slash (idx and count are equal and equal to the whole string)
-    case "$shard_idx" in
-        ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;;
-    esac
-    case "$shard_count" in
-        ''|*[!0-9]*|0*) echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1 ;;
-    esac
-    # Confirm the string is exactly "idx/count" — catches "12" (no slash) and
-    # "1/2/3" (extra slash, where idx=1 and count=2/3 would have failed above,
-    # but this is an extra safety net).
-    if [ "$shard" != "$shard_idx/$shard_count" ]; then
-        echo "ERROR: bad shard '$shard' (expected i/N)"; exit 1
-    fi
-    if [ "$shard_idx" -lt 1 ] || [ "$shard_idx" -gt "$shard_count" ]; then
-        echo "ERROR: bad shard '$shard' (expected i/N with 1 <= i <= N)"; exit 1
-    fi
-fi
+shard_idx="$caseopt_shard_idx"
+shard_count="$caseopt_shard_count"
 
 # Phoenix starts fresh (no prior dep build); other clusters pre-build deps via
 # build.sh first, so we must preserve them and only clean MFC target staging.
@@ -60,8 +42,8 @@ if [ "$cluster" = "phoenix" ]; then
     source .github/scripts/clean-build.sh
     clean_build
 elif [ -z "$shard" ]; then
-    find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
-    find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/staging -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/install -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true
 fi
 
 . ./mfc.sh load -c "$flag" -m g
@@ -80,8 +62,7 @@ esac
 if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then
     shared_marker_done="build/.prebuild-shared-targets-done"
     shared_marker_failed="build/.prebuild-shared-targets-failed"
-    set -- benchmarks/*/case.py
-    first_case="$1"
+    first_case="${benchmarks[0]}"
     if [ "$shard_idx" -eq 1 ]; then
         # Remove both markers at the start so reruns and manual invocations
         # never observe stale state from a prior run.
@@ -110,11 +91,9 @@ if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then
 fi
 
 idx=0
-for case in benchmarks/*/case.py; do
+for case in "${benchmarks[@]}"; do
     idx=$((idx + 1))
-    if [ -n "$shard" ] && [ $(((idx - 1) % shard_count)) -ne $((shard_idx - 1)) ]; then
-        continue
-    fi
+    caseopt_case_in_shard "$idx" || continue
     echo "=== Pre-building: $case ==="
     ./mfc.sh run "$case" --case-optimization $gpu_opts -j 8 --dry-run
 done

.github/scripts/run_case_optimization.sh

@@ -13,13 +13,11 @@ if [ "$job_device" = "gpu" ] && [ "$ngpus" -eq 0 ]; then
     ngpus=1
 fi
 
-benchmarks=(
-    benchmarks/5eq_rk3_weno3_hllc/case.py
-    benchmarks/viscous_weno5_sgb_acoustic/case.py
-    benchmarks/hypo_hll/case.py
-    benchmarks/ibm/case.py
-    benchmarks/igr/case.py
-)
+# Benchmark list (single source of truth shared with the pre-build) + optional
+# sharding: a sharded run executes only the subset it has pre-built binaries
+# for. $job_shard mirrors the pre-build shard so the two stay aligned.
+source .github/scripts/case-optimization-benchmarks.sh
+caseopt_parse_shard
 
 # For Frontier/Frontier AMD: deps were fetched on the login node via --deps-only;
 # build case-optimized binaries here on the compute node before running.
@@ -35,11 +33,14 @@ if [ "$job_cluster" != "phoenix" ] && [ "$job_cluster" != "frontier_amd" ]; then
     # preserve dependency dirs (hipfort, fftw, etc.) since the compute
     # node has no internet to re-fetch them.
     echo "=== Cleaning stale MFC target staging/install ==="
-    find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
-    find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/staging -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/install -maxdepth 1 -regex '.*/\(gpu-acc\|gpu-mp\|cpu\)-.*' -type d -exec rm -rf {} + 2>/dev/null || true
 
     echo "=== Building case-optimized binaries on compute node ==="
+    idx=0
     for case in "${benchmarks[@]}"; do
+        idx=$((idx + 1))
+        caseopt_case_in_shard "$idx" || continue
         echo "--- Building: $case ---"
         ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8
     done
@@ -50,7 +51,10 @@ passed=0
 failed=0
 failed_cases=""
 
+idx=0
 for case in "${benchmarks[@]}"; do
+    idx=$((idx + 1))
+    caseopt_case_in_shard "$idx" || continue
     case_dir="$(dirname "$case")"
     case_name="$(basename "$case_dir")"
     echo ""

.github/workflows/bench.yml

@@ -38,7 +38,7 @@ jobs:
       matrix:
         include:
           - cluster: phoenix
-            name: Georgia Tech | Phoenix (NVHPC)
+            name: Georgia Tech | Phoenix (GNU)
             group: phoenix
             labels: gt
             flag: p