CI: Emit JUnit from test runners, pin their exec bounds

hack/run-e2e.sh

@@ -51,6 +51,14 @@ Common Go Test Flags (passed before -args):
   -run <regexp>    Run only tests matching regexp
   -v               Verbose output
   -count n         Run tests n times
+  -p n             Test package concurrency (default: \$E2E_PARALLELISM, or 4)
+  -timeout d       Per-binary timeout (default: \$E2E_TIMEOUT, or 30m)
+
+Passing -p or -timeout explicitly overrides the default for that flag.
+
+Environment Variables:
+  E2E_PARALLELISM  Default for -p (default: 4)
+  E2E_TIMEOUT      Default for -timeout (default: 30m)
 
 See "go help testflag" for more Go test flags.
 EOF
@@ -81,6 +89,8 @@ fi
 go_test_args=()
 e2e_args=()
 found_args_sep=false
+has_p_flag=false
+has_timeout_flag=false
 
 for arg in "$@"; do
     if [[ "$arg" == "-h" || "$arg" == "--help" ]]; then
@@ -95,6 +105,11 @@ for arg in "$@"; do
     if [[ "$found_args_sep" == "true" ]]; then
         e2e_args+=("$arg")
     else
+        # Both spellings: go's flag package accepts -flag, --flag, and either with =value.
+        case "$arg" in
+            -p|-p=*|--p|--p=*) has_p_flag=true ;;
+            -timeout|-timeout=*|--timeout|--timeout=*) has_timeout_flag=true ;;
+        esac
         go_test_args+=("$arg")
     fi
 done
@@ -104,5 +119,47 @@ if [[ -n "${KUBECTL_CONTEXT:-}" ]]; then
     extra_e2e_args+=("--kube-context" "${KUBECTL_CONTEXT}")
 fi
 
-exec go test -v "$target_path" ${go_test_args[@]+"${go_test_args[@]}"} -args --e2e ${extra_e2e_args[@]+"${extra_e2e_args[@]}"} ${e2e_args[@]+"${e2e_args[@]}"}
+# Pin the two bounds go test would otherwise infer from the machine.
+#
+# -p: the system default is GOMAXPROCS causing the suite concurrency to track the
+# runner's CPU count rather than what the cluster can absorb, in this case a single
+# node Kind cluster. Explicitly setting this to E2E_PARALLELISM (default 4) avoids
+# overshooting the cluster's capacity.
+#
+# -timeout: Go's default, when no value is provided, is 10m. TemplateReadyTimeout
+# for the micro-VM class (internal/e2e/sandbox.go) is also 10 minutes. Any E2E
+# test that times out waiting for the template will actually be killed by the global
+# suite timeout instead, which skips t.Cleanup, potentially leaking resources.
+# Explicitly setting the E2E_TIMEOUT (default 30m) here avoids this suite-level and
+# test-level timeout conflict.
+default_go_test_args=()
+if [[ "${has_p_flag}" == "false" ]]; then
+    default_go_test_args+=("-p" "${E2E_PARALLELISM:-4}")
+fi
+if [[ "${has_timeout_flag}" == "false" ]]; then
+    default_go_test_args+=("-timeout" "${E2E_TIMEOUT:-30m}")
+fi
+
+# Assembled once so the two execution paths below cannot drift apart.
+test_argv=(-v "$target_path")
+test_argv+=(${default_go_test_args[@]+"${default_go_test_args[@]}"})
+test_argv+=(${go_test_args[@]+"${go_test_args[@]}"})
+test_argv+=(-args --e2e)
+test_argv+=(${extra_e2e_args[@]+"${extra_e2e_args[@]}"})
+test_argv+=(${e2e_args[@]+"${e2e_args[@]}"})
+
+# E2E_JUNIT_FILE opts into a machine-readable record of the run: the XML for
+# report consumers, the JSON event stream for failure analysis. Unset, this is a
+# plain go test and gotestsum is never built, so a local run needs no toolchain
+# beyond go itself.
+if [[ -n "${E2E_JUNIT_FILE:-}" ]]; then
+    mkdir -p "$(dirname "${E2E_JUNIT_FILE}")"
+    exec "${ROOT}/hack/run-tool.sh" gotestsum \
+        --junitfile "${E2E_JUNIT_FILE}" \
+        --jsonfile "${E2E_JUNIT_FILE%.xml}.json" \
+        --format standard-verbose \
+        -- "${test_argv[@]}"
+fi
+
+exec go test "${test_argv[@]}"
 

hack/run-root-tests.sh

@@ -42,10 +42,30 @@ echo "${PKGS}"
 
 # -count=1: the Go test cache does not key on euid, so without it a rerun as
 # root replays the unprivileged run's cached skips.
+# -timeout: declared here rather than inherited from go's 10m default, and
+# overridable for a slow machine.
+test_args=(-count=1 -timeout "${ROOT_TEST_TIMEOUT:-10m}" "$@")
+
+# ROOT_JUNIT_FILE opts into a machine-readable record of the run. Resolve the
+# binary as the invoking user: hack/run-tool.sh compiles on demand, and doing
+# that under sudo would leave root-owned entries in the user's build cache.
+# The runner is only ever a prefix — the privilege dispatch below is unchanged,
+# because a root-gated package run without root self-skips and reports success.
+if [[ -n "${ROOT_JUNIT_FILE:-}" ]]; then
+  mkdir -p "$(dirname "${ROOT_JUNIT_FILE}")"
+  runner=("$("${ROOT}/hack/run-tool.sh" --print-bin-path gotestsum)"
+    --junitfile "${ROOT_JUNIT_FILE}"
+    --jsonfile "${ROOT_JUNIT_FILE%.xml}.json"
+    --format standard-verbose
+    --)
+else
+  runner=(go test)
+fi
+
 # shellcheck disable=SC2086 # intentional word splitting of the package list
 if [[ "$(id -u)" -eq 0 ]]; then
-  exec go test -count=1 "$@" ${PKGS}
+  exec "${runner[@]}" "${test_args[@]}" ${PKGS}
 fi
 # -E / env PATH: keep the invoking user's Go toolchain and module caches.
 # shellcheck disable=SC2086
-exec sudo -E env "PATH=${PATH}" go test -count=1 "$@" ${PKGS}
+exec sudo -E env "PATH=${PATH}" "${runner[@]}" "${test_args[@]}" ${PKGS}

hack/tools/gotestsum/go.mod

@@ -0,0 +1,22 @@
+module github.com/agent-substrate/substrate/hack/tools/gotestsum
+
+go 1.27.0
+
+tool gotest.tools/gotestsum
+
+require (
+	github.com/bitfield/gotestdox v0.2.2 // indirect
+	github.com/dnephin/pflag v1.0.7 // indirect
+	github.com/fatih/color v1.18.0 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	golang.org/x/mod v0.27.0 // indirect
+	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/sys v0.36.0 // indirect
+	golang.org/x/term v0.35.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
+	golang.org/x/tools v0.36.0 // indirect
+	gotest.tools/gotestsum v1.13.0 // indirect
+)

hack/tools/gotestsum/go.sum

@@ -0,0 +1,37 @@
+github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE=
+github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY=
+github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk=
+github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
+golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
+golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
+golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
+gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo=
+gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q=
+gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=