add spack retry to CI

share/ramble/cloud-build/Dockerfile-apt

@@ -21,6 +21,7 @@ RUN cd /opt &&  \
     python -m pip install --no-cache-dir -r /opt/requirements-pinned.txt
 COPY share/ramble/cloud-build/container_test.sh /container_test.sh
 RUN chmod a+x /container_test.sh
+RUN chmod a+rx /root && ([ ! -d /root/.spack ] || chmod -R a+rX /root/.spack)
 FROM ${BASE_IMG}:${BASE_VER}
 
 COPY --from=builder / /

share/ramble/cloud-build/Dockerfile-yum

@@ -22,6 +22,7 @@ RUN cd /opt &&  \
     python -m pip install --no-cache-dir -r /opt/requirements-pinned.txt
 COPY share/ramble/cloud-build/container_test.sh /container_test.sh
 RUN chmod a+x /container_test.sh
+RUN chmod a+rx /root && ([ ! -d /root/.spack ] || chmod -R a+rX /root/.spack)
 FROM ${BASE_IMG}:${BASE_VER}
 
 COPY --from=builder / /

share/ramble/cloud-build/ramble-perf-tests.yaml

@@ -28,22 +28,7 @@ steps:
     args:
       - '-c'
       - |
-        cd /workspace
-
-        git branch develop origin/develop
-
-        . /opt/spack/share/spack/setup-env.sh
-
-        spack load py-pip ^python
-
-        pip install -r /workspace/requirements-pinned.txt
-
-        export SPACK_PYTHON=$(which python3)
-
-        . /workspace/share/ramble/setup-env.sh
-
-        echo "Spack version is $(spack --version)"
-        echo "Python version is $(python3 --version)"
+        . /workspace/share/ramble/cloud-build/setup-ci-env.sh
 
         echo "Running Perf Tests..."
         ramble unit-test --perf

share/ramble/cloud-build/ramble-pr-docs.yaml

@@ -32,22 +32,7 @@ steps:
       - '-c'
       - |
         set -e
-        cd /workspace
-
-        git branch develop origin/develop
-
-        . /opt/spack/share/spack/setup-env.sh
-
-        spack load py-pip ^python
-
-        pip install -r /workspace/requirements-pinned.txt
-
-        export SPACK_PYTHON=`which python3`
-
-        . /workspace/share/ramble/setup-env.sh
-
-        echo "Spack version is $(spack --version)"
-        echo "Python version is $(python3 --version)"
+        . /workspace/share/ramble/cloud-build/setup-ci-env.sh
 
         cd lib/ramble/docs
 

share/ramble/cloud-build/ramble-pr-software-conflicts.yaml

@@ -31,22 +31,7 @@ steps:
     args:
       - '-c'
       - |
-        cd /workspace
-
-        git branch develop origin/develop
-
-        . /opt/spack/share/spack/setup-env.sh
-
-        spack load py-pip ^python
-
-        pip install -r /workspace/requirements-pinned.txt
-
-        export SPACK_PYTHON=`which python3`
-
-        . /workspace/share/ramble/setup-env.sh
-
-        echo "Spack version is $(spack --version)"
-        echo "Python version is $(python3 --version)"
+        . /workspace/share/ramble/cloud-build/setup-ci-env.sh
 
         ramble software-definitions -s
 

share/ramble/cloud-build/ramble-pr-style.yaml

@@ -31,22 +31,7 @@ steps:
     args:
       - '-c'
       - |
-        cd /workspace
-
-        git branch develop origin/develop
-
-        . /opt/spack/share/spack/setup-env.sh
-
-        spack load py-pip ^python
-
-        pip install -r /workspace/requirements-pinned.txt
-
-        export SPACK_PYTHON=`which python3`
-
-        . /workspace/share/ramble/setup-env.sh
-
-        echo "Spack version is $(spack --version)"
-        echo "Python version is $(python3 --version)"
+        . /workspace/share/ramble/cloud-build/setup-ci-env.sh
 
         ramble style -a -U
         # $$ characters are required for cloud-build:

share/ramble/cloud-build/ramble-pr-unit-tests.yaml

@@ -31,22 +31,7 @@ steps:
     args:
       - '-c'
       - |
-        cd /workspace
-
-        git branch develop origin/develop
-
-        . /opt/spack/share/spack/setup-env.sh
-
-        spack load py-pip ^python
-
-        pip install -r /workspace/requirements-pinned.txt
-
-        export SPACK_PYTHON=`which python3`
-
-        . /workspace/share/ramble/setup-env.sh
-
-        echo "Spack version is $(spack --version)"
-        echo "Python version is $(python3 --version)"
+        . /workspace/share/ramble/cloud-build/setup-ci-env.sh
 
         UNIT_TEST_COVERAGE=true LONG=true /workspace/share/ramble/qa/run-unit-tests
         # $$ characters are required for cloud-build:

share/ramble/cloud-build/setup-ci-env.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright 2022-2026 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+cd /workspace
+
+# Ensure develop branch exists for comparison in PR builds
+if ! git rev-parse --verify develop >/dev/null 2>&1; then
+  git branch develop origin/develop 2>/dev/null || true
+fi
+
+. /opt/spack/share/spack/setup-env.sh
+
+# Pre-seed cache from container image if available to avoid cloning over network
+if [ -d /root/.spack ] && [ ! -d "$HOME/.spack" ]; then
+  cp -rn /root/.spack "$HOME/.spack" 2>/dev/null || true
+fi
+
+# Load py-pip with retries against transient git/network failures
+spack_loaded=0
+for attempt in 1 2 3; do
+  if spack load py-pip ^python; then
+    spack_loaded=1
+    break
+  fi
+  echo "WARNING: 'spack load py-pip ^python' failed on attempt $attempt of 3."
+  rm -rf "$HOME/.spack/package_repos"
+  sleep 5
+done
+
+if [ $spack_loaded -eq 0 ] || ! command -v pip >/dev/null 2>&1; then
+  echo "======================================================================"
+  echo "FATAL: Infrastructure Failure - Failed to initialize Spack environment."
+  echo "'spack load py-pip ^python' failed after 3 attempts."
+  echo "This is caused by transient external network issues reaching GitHub"
+  echo "(https://github.com/spack/spack-packages.git)."
+  echo "======================================================================"
+  exit 1
+fi
+
+pip install -r /workspace/requirements-pinned.txt
+if [ $? -ne 0 ]; then
+  echo "FATAL: Failed to install pinned requirements via pip."
+  exit 1
+fi
+
+export SPACK_PYTHON=$(which python3)
+
+. /workspace/share/ramble/setup-env.sh
+
+echo "Spack version is $(spack --version)"
+echo "Python version is $(python3 --version)"