Repository navigation
[Nightly] RHEL 9.6 @ cron #157
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "CRCR Nightly: RHEL 9.6 Build & Test" | |
| on: | |
| schedule: | |
| - cron: "30 8 * * *" # daily at 08:30 UTC, ~30 min after pytorch/pytorch nightly update | |
| workflow_dispatch: | |
| inputs: | |
| sha: | |
| description: "pytorch/pytorch SHA to build against (leave empty for latest nightly)" | |
| required: false | |
| default: "" | |
| test_categories: | |
| description: "Which test stages to run after build" | |
| required: false | |
| type: choice | |
| default: "all" | |
| options: | |
| - all | |
| - cpu | |
| - inductor | |
| - sgpu | |
| - mgpu | |
| - build-only | |
| run-name: >- | |
| [Nightly] RHEL 9.6 @ ${{ github.event_name == 'schedule' && 'cron' || inputs.sha && 'manual-' || 'manual' }}${{ inputs.sha && format(' ({0})', inputs.sha) || '' }}${{ inputs.test_categories && inputs.test_categories != 'all' && format(' [{0}]', inputs.test_categories) || '' }} | |
| env: | |
| UPSTREAM_REPO: pytorch/pytorch | |
| NIGHTLY_BRANCH: nightly | |
| QUAY_REGISTRY: quay.io/aipcc/pytorch | |
| CUDA_VERSION: "13_0" | |
| # Test tier: "sanity", "critical", "full", or "delta" | |
| # delta = diff-based determination merged with critical tests | |
| TEST_TIER: "critical" | |
| permissions: | |
| contents: read | |
| id-token: write | |
| jobs: | |
| build: | |
| name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / build" | |
| runs-on: linux.rhel96 | |
| timeout-minutes: 600 | |
| outputs: | |
| sha: ${{ steps.sha.outputs.sha }} | |
| prev_sha: ${{ steps.sha.outputs.prev_sha }} | |
| image_tag: ${{ steps.sha.outputs.image_tag }} | |
| quay_tag: ${{ steps.sha.outputs.quay_tag }} | |
| steps: | |
| - name: Checkout CI repo | |
| uses: actions/checkout@v4 | |
| - name: Fetch source SHAs from nightly branch | |
| id: sha | |
| env: | |
| INPUT_SHA: ${{ inputs.sha }} | |
| run: | | |
| if [ -n "$INPUT_SHA" ]; then | |
| echo "Using manually provided SHA: $INPUT_SHA" | |
| SOURCE_SHA="$INPUT_SHA" | |
| PREV_SOURCE_SHA="" | |
| else | |
| NIGHTLY_COMMITS=$(curl -fsSL "https://api.github.com/repos/${UPSTREAM_REPO}/commits?sha=${NIGHTLY_BRANCH}&per_page=2" | jq -r '.[].sha') | |
| NIGHTLY_SHA=$(echo "$NIGHTLY_COMMITS" | head -1) | |
| PREV_NIGHTLY_SHA=$(echo "$NIGHTLY_COMMITS" | tail -1) | |
| if [ -z "$NIGHTLY_SHA" ]; then | |
| echo "::error::Could not resolve HEAD for ${UPSTREAM_REPO}/${NIGHTLY_BRANCH}" | |
| exit 1 | |
| fi | |
| extract_source_sha() { | |
| local COMMIT_MSG=$(curl -fsSL "https://api.github.com/repos/${UPSTREAM_REPO}/commits/$1" | jq -r '.commit.message') | |
| echo "$COMMIT_MSG" | grep -oP '\(([a-f0-9]{40})\)' | tr -d '()' | |
| } | |
| SOURCE_SHA=$(extract_source_sha "$NIGHTLY_SHA") | |
| if [ -z "$SOURCE_SHA" ]; then | |
| echo "::error::Could not extract source SHA from latest nightly commit" | |
| exit 1 | |
| fi | |
| PREV_SOURCE_SHA=$(extract_source_sha "$PREV_NIGHTLY_SHA") | |
| if [ -z "$PREV_SOURCE_SHA" ]; then | |
| echo "::warning::Could not extract previous source SHA, will run full test suite" | |
| PREV_SOURCE_SHA="" | |
| fi | |
| fi | |
| echo "Source main SHA (current): $SOURCE_SHA" | |
| echo "Source main SHA (previous): $PREV_SOURCE_SHA" | |
| SHORT_SHA=${SOURCE_SHA:0:7} | |
| QUAY_TAG="${QUAY_REGISTRY}:rhel9_6_pytorch_nightly_main_git${SHORT_SHA}_cuda${CUDA_VERSION}" | |
| echo "sha=$SOURCE_SHA" >> "$GITHUB_OUTPUT" | |
| echo "prev_sha=$PREV_SOURCE_SHA" >> "$GITHUB_OUTPUT" | |
| echo "image_tag=rhel9-pytorch-nightly:${SOURCE_SHA}" >> "$GITHUB_OUTPUT" | |
| echo "quay_tag=$QUAY_TAG" >> "$GITHUB_OUTPUT" | |
| echo "Quay tag: $QUAY_TAG" | |
| - name: Log in to Quay.io | |
| env: | |
| QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} | |
| QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} | |
| run: | | |
| echo "$QUAY_PASSWORD" | podman login quay.io -u "$QUAY_USERNAME" --password-stdin | |
| - name: Build container image | |
| env: | |
| IMAGE_TAG: ${{ steps.sha.outputs.image_tag }} | |
| PYTORCH_SHA: ${{ steps.sha.outputs.sha }} | |
| SUBSCRIPTION_KEY: ${{ secrets.RHEL_SUBSCRIPTION_ACTIVATION_KEY }} | |
| SUBSCRIPTION_ORG: ${{ secrets.RHEL_SUBSCRIPTION_ORG_ID }} | |
| run: | | |
| echo "$SUBSCRIPTION_KEY" > /tmp/sub_key.txt | |
| echo "$SUBSCRIPTION_ORG" > /tmp/sub_org.txt | |
| podman build \ | |
| --file docker/Dockerfile.rhel9 \ | |
| --build-arg COMMIT_HASH="${PYTORCH_SHA}" \ | |
| --build-arg ENABLE_SUBSCRIPTION=true \ | |
| --build-arg RHEL_OS_VERSION=9.6 \ | |
| --secret id=subscription_key,src=/tmp/sub_key.txt \ | |
| --secret id=subscription_org,src=/tmp/sub_org.txt \ | |
| --tag "${IMAGE_TAG}" \ | |
| . | |
| rm -f /tmp/sub_key.txt /tmp/sub_org.txt | |
| - name: Tag and push to Quay.io | |
| id: quay_push | |
| continue-on-error: true | |
| env: | |
| IMAGE_TAG: ${{ steps.sha.outputs.image_tag }} | |
| QUAY_TAG: ${{ steps.sha.outputs.quay_tag }} | |
| run: | | |
| podman tag "${IMAGE_TAG}" "${QUAY_TAG}" | |
| podman push "${QUAY_TAG}" | |
| echo "Pushed: ${QUAY_TAG}" | |
| - name: Build summary | |
| env: | |
| IMAGE_TAG: ${{ steps.sha.outputs.image_tag }} | |
| QUAY_TAG: ${{ steps.sha.outputs.quay_tag }} | |
| PYTORCH_SHA: ${{ steps.sha.outputs.sha }} | |
| run: | | |
| echo "## CRCR Nightly: RHEL 9.6 — Build" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Nightly SHA | [\`${PYTORCH_SHA}\`](https://github.com/pytorch/pytorch/commit/${PYTORCH_SHA}) |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Local Image | \`${IMAGE_TAG}\` |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ steps.quay_push.outcome }}" = "success" ]; then | |
| echo "| Quay Image | \`${QUAY_TAG}\` |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "::warning::Failed to upload ${QUAY_TAG} to Quay.io" | |
| echo "| Quay Image | failed to upload \`${QUAY_TAG}\` |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Report to CRCR | |
| if: always() && job.status == 'success' | |
| continue-on-error: true | |
| uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main | |
| with: | |
| status: completed | |
| conclusion: ${{ job.status }} | |
| delivery-id: ${{ steps.sha.outputs.sha }} | |
| event-type: nightly | |
| job-name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / build" | |
| determine-tests: | |
| name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / determine-tests" | |
| needs: build | |
| if: ${{ !cancelled() && needs.build.result == 'success' && (github.event_name == 'schedule' || inputs.test_categories != 'build-only') }} | |
| runs-on: linux.rhel96 | |
| timeout-minutes: 600 | |
| outputs: | |
| cpu_tests: ${{ steps.resolve.outputs.cpu_tests }} | |
| inductor_tests: ${{ steps.resolve.outputs.inductor_tests }} | |
| sgpu_tests: ${{ steps.resolve.outputs.sgpu_tests }} | |
| mgpu_tests: ${{ steps.resolve.outputs.mgpu_tests }} | |
| mode: ${{ steps.resolve.outputs.mode }} | |
| env: | |
| IMAGE_TAG: ${{ needs.build.outputs.image_tag }} | |
| PYTORCH_SHA: ${{ needs.build.outputs.sha }} | |
| PREV_SHA: ${{ needs.build.outputs.prev_sha }} | |
| steps: | |
| - name: Checkout CI repo | |
| uses: actions/checkout@v4 | |
| - name: Resolve test lists | |
| id: resolve | |
| run: | | |
| MODE="full" | |
| # Tests that unconditionally require a working CUDA driver | |
| # (they call torch.Stream() or similar without guarding) | |
| CUDA_ONLY_TESTS="test_overrides" | |
| if [ "${TEST_TIER}" = "delta" ] && [ -n "${PREV_SHA}" ]; then | |
| echo "Delta mode: comparing ${PREV_SHA:0:10}..${PYTORCH_SHA:0:10}" | |
| # Heuristic + structural analysis (C++ call graph is | |
| # pre-built and cached inside the image after the first run) | |
| ALL_RESULTS=$(podman run --rm -v "$(pwd)/scripts:/ci_scripts:ro" "${IMAGE_TAG}" bash -c " | |
| source /miniconda/etc/profile.d/conda.sh && conda activate cuda_torch_build | |
| cd /pytorch | |
| # Build valid test name list from run_test.py | |
| VALID_TESTS=\$(python test/run_test.py -i __probe__ 2>&1 | grep 'invalid choice' | sed \"s/.*choose from //\" | tr -d \")'\" | tr ',' '\n' | sed 's/^ //' || true) | |
| for cat in cpu inductor sgpu mgpu; do | |
| RAW=\$(python /ci_scripts/merge_test_results.py '${PREV_SHA}' '${PYTORCH_SHA}' --pytorch-dir /pytorch --category \$cat --commands-only 2>/dev/null) || true | |
| # Filter: keep only commands whose -i test name is in VALID_TESTS | |
| CMDS=\"\" | |
| while IFS= read -r line; do | |
| [ -z \"\$line\" ] && continue | |
| TEST_NAME=\$(echo \"\$line\" | sed -n 's/.*-i \([^ ]*\).*/\1/p') | |
| if [ -z \"\$TEST_NAME\" ] || echo \"\$VALID_TESTS\" | grep -qxF \"\$TEST_NAME\"; then | |
| CMDS=\"\${CMDS}\${line}\n\" | |
| fi | |
| done <<< \"\$RAW\" | |
| echo \":::\${cat}:::\" | |
| echo -e \"\$CMDS\" | |
| done | |
| ") || true | |
| echo "Raw output from container:" | |
| echo "$ALL_RESULTS" | |
| # Parse results by category delimiter | |
| for cat in cpu inductor sgpu mgpu; do | |
| CMDS=$(echo "$ALL_RESULTS" | sed -n "/:::${cat}:::/,/:::.*:::/{ /:::/d; p; }" | sed '/^$/d') | |
| # Exclude CUDA-only tests from cpu/inductor categories | |
| if [ "$cat" = "cpu" ] || [ "$cat" = "inductor" ]; then | |
| FILTERED="" | |
| while IFS= read -r line; do | |
| [ -z "$line" ] && continue | |
| SKIP=false | |
| for excl in $CUDA_ONLY_TESTS; do | |
| if echo "$line" | grep -q "\-i ${excl}"; then | |
| SKIP=true | |
| break | |
| fi | |
| done | |
| if [ "$SKIP" = "false" ]; then | |
| FILTERED="${FILTERED}${line}\n" | |
| fi | |
| done <<< "$CMDS" | |
| CMDS=$(echo -e "$FILTERED" | sed '/^$/d') | |
| fi | |
| if [ -n "$CMDS" ]; then | |
| MODE="delta" | |
| fi | |
| echo "${cat}_tests=$(echo "$CMDS" | base64 -w0)" >> "$GITHUB_OUTPUT" | |
| done | |
| fi | |
| # Resolve final test list per category | |
| TIER_FLAG="" | |
| case "${TEST_TIER}" in | |
| sanity) TIER_FLAG="--sanity" ;; | |
| critical) TIER_FLAG="--critical" ;; | |
| full) TIER_FLAG="" ;; | |
| delta) TIER_FLAG="--critical" ;; # fallback when delta produces nothing | |
| esac | |
| for cat in cpu inductor sgpu mgpu; do | |
| VAR="${cat}_tests" | |
| CURRENT=$(grep "^${VAR}=" "$GITHUB_OUTPUT" 2>/dev/null | cut -d= -f2-) | |
| DECODED=$(echo "$CURRENT" | base64 -d 2>/dev/null || true) | |
| if [ -z "$DECODED" ]; then | |
| echo "No delta tests for ${cat}, using ${TEST_TIER} tier" | |
| if [ -n "$TIER_FLAG" ]; then | |
| MERGED=$(python3 scripts/test_config.py "${cat}" ${TIER_FLAG} --commands-only) | |
| else | |
| MERGED=$(python3 scripts/test_config.py "${cat}" --commands-only) | |
| fi | |
| else | |
| # Prepend critical tests to delta results, then dedup | |
| CRITICAL=$(python3 scripts/test_config.py "${cat}" --critical --commands-only) | |
| MERGED=$(printf '%s\n%s' "$CRITICAL" "$DECODED" | awk '!seen[$0]++' | sed '/^$/d') | |
| fi | |
| echo "${cat}_tests=$(echo "$MERGED" | base64 -w0)" >> "$GITHUB_OUTPUT" | |
| done | |
| echo "mode=${MODE}" >> "$GITHUB_OUTPUT" | |
| - name: Determination summary | |
| env: | |
| CPU_B64: ${{ steps.resolve.outputs.cpu_tests }} | |
| INDUCTOR_B64: ${{ steps.resolve.outputs.inductor_tests }} | |
| SGPU_B64: ${{ steps.resolve.outputs.sgpu_tests }} | |
| MGPU_B64: ${{ steps.resolve.outputs.mgpu_tests }} | |
| run: | | |
| echo "## CRCR Nightly: Test Determination" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Mode | \`${{ steps.resolve.outputs.mode }}\` |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Strategy | heuristic + structural (union) |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Current SHA | \`${PYTORCH_SHA:0:10}\` |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Previous SHA | \`${PREV_SHA:0:10}\` |" >> $GITHUB_STEP_SUMMARY | |
| for pair in "cpu:$CPU_B64" "inductor:$INDUCTOR_B64" "sgpu:$SGPU_B64" "mgpu:$MGPU_B64"; do | |
| NAME="${pair%%:*}" | |
| B64="${pair#*:}" | |
| COUNT=$(echo "$B64" | base64 -d 2>/dev/null | grep -c '.' || echo 0) | |
| echo "| ${NAME} tests | ${COUNT} |" >> $GITHUB_STEP_SUMMARY | |
| done | |
| cpu-tests: | |
| name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (cpu, linux.rhel96)" | |
| needs: [build, determine-tests] | |
| if: ${{ !cancelled() && needs.build.result == 'success' && needs.determine-tests.result == 'success' && (github.event_name == 'schedule' || inputs.test_categories == 'all' || inputs.test_categories == 'cpu') }} | |
| runs-on: linux.rhel96 | |
| timeout-minutes: 1440 | |
| env: | |
| IMAGE_TAG: ${{ needs.build.outputs.image_tag }} | |
| PYTORCH_SHA: ${{ needs.build.outputs.sha }} | |
| steps: | |
| - name: Verify image exists | |
| run: | | |
| if ! podman image exists "${IMAGE_TAG}"; then | |
| echo "::error::Image ${IMAGE_TAG} not found in local podman store" | |
| exit 1 | |
| fi | |
| - name: Quick sanity gate | |
| run: | | |
| podman run --rm "${IMAGE_TAG}" bash -c ' | |
| source /miniconda/etc/profile.d/conda.sh && conda activate cuda_torch_build | |
| cd /tmp | |
| python -c " | |
| import torch, tempfile, os | |
| print(f\"PyTorch {torch.__version__} loaded\") | |
| z = torch.randn(64, 64) @ torch.randn(64, 64) | |
| assert z.shape == (64, 64) | |
| for dt in [torch.float32, torch.float64, torch.int32, torch.int64, torch.bfloat16]: | |
| assert torch.ones(8, dtype=dt).dtype == dt | |
| x = torch.randn(10, requires_grad=True) | |
| ((x ** 2).sum()).backward() | |
| assert x.grad is not None and x.grad.shape == x.shape | |
| m = torch.nn.Linear(16, 4) | |
| with tempfile.NamedTemporaryFile(suffix=\".pt\", delete=False) as f: | |
| torch.save(m.state_dict(), f.name) | |
| torch.nn.Linear(16, 4).load_state_dict(torch.load(f.name, weights_only=True)) | |
| os.unlink(f.name) | |
| print(\"Quick sanity gate passed\") | |
| " | |
| ' | |
| - name: Run CPU test suite | |
| id: cpu_run | |
| continue-on-error: true | |
| env: | |
| TEST_LIST_B64: ${{ needs.determine-tests.outputs.cpu_tests }} | |
| run: | | |
| COMMANDS=$(echo "$TEST_LIST_B64" | base64 -d) | |
| if [ -z "$COMMANDS" ]; then | |
| echo "No CPU tests to run" | |
| echo "passed=0" >> "$GITHUB_OUTPUT" | |
| echo "failed=0" >> "$GITHUB_OUTPUT" | |
| echo "errors=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| CMD_COUNT=$(echo "$COMMANDS" | wc -l) | |
| echo "Running ${CMD_COUNT} CPU test command(s)" | |
| LOG=/tmp/cpu_test_output.log | |
| echo "$COMMANDS" > /tmp/cpu_test_commands.txt | |
| podman run --rm \ | |
| -e CONTINUE_THROUGH_ERROR=True \ | |
| -v /tmp/cpu_test_commands.txt:/tmp/test_commands.txt:ro \ | |
| "${IMAGE_TAG}" bash -c ' | |
| source /miniconda/etc/profile.d/conda.sh && conda activate cuda_torch_build | |
| cd /pytorch | |
| PASSED=0; FAILED=0; ERRORS="" | |
| while IFS= read -r cmd; do | |
| [ -z "$cmd" ] && continue | |
| echo "" | |
| echo "============================================================" | |
| echo "Running: $cmd" | |
| echo "============================================================" | |
| echo "$cmd" > /tmp/_run.sh | |
| if timeout 7200 bash /tmp/_run.sh; then | |
| PASSED=$((PASSED + 1)) | |
| else | |
| RC=$? | |
| FAILED=$((FAILED + 1)) | |
| if [ $RC -eq 124 ]; then | |
| ERRORS="${ERRORS}\n TIMEOUT(2h): $cmd" | |
| else | |
| ERRORS="${ERRORS}\n FAIL: $cmd" | |
| fi | |
| fi | |
| done < /tmp/test_commands.txt | |
| echo "" | |
| echo ":::SUMMARY:::" | |
| echo "PASSED=${PASSED}" | |
| echo "FAILED=${FAILED}" | |
| if [ $FAILED -gt 0 ]; then | |
| echo -e "ERRORS=${ERRORS}" | |
| fi | |
| ' 2>&1 | tee "$LOG" || true | |
| PASSED=$(grep '^PASSED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAILED=$(grep '^FAILED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAIL_LIST=$(grep '^\s*\(FAIL\|TIMEOUT\)' "$LOG" || true) | |
| echo "passed=${PASSED:-0}" >> "$GITHUB_OUTPUT" | |
| echo "failed=${FAILED:-0}" >> "$GITHUB_OUTPUT" | |
| { | |
| echo 'errors<<ERRORS_EOF' | |
| echo "$FAIL_LIST" | |
| echo 'ERRORS_EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Test summary | |
| if: always() | |
| env: | |
| ERRORS_OUTPUT: ${{ steps.cpu_run.outputs.errors }} | |
| run: | | |
| echo "## CRCR Nightly: RHEL 9.6 — CPU Tests" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Nightly SHA | [\`${PYTORCH_SHA}\`](https://github.com/pytorch/pytorch/commit/${PYTORCH_SHA}) |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Mode | \`${{ needs.determine-tests.outputs.mode }}\` |" >> $GITHUB_STEP_SUMMARY | |
| FAILED="${{ steps.cpu_run.outputs.failed || '0' }}" | |
| echo "| Passed | ${{ steps.cpu_run.outputs.passed || '—' }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Failed | ${FAILED} |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${FAILED}" -gt 0 ] 2>/dev/null; then | |
| echo "| Result | \`completed with failures\` |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| Result | \`success\` |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ -n "$ERRORS_OUTPUT" ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Failed Commands" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "$ERRORS_OUTPUT" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Determine conclusion | |
| id: conclusion | |
| if: always() | |
| env: | |
| FAILED: ${{ steps.cpu_run.outputs.failed }} | |
| run: | | |
| if [ -n "$FAILED" ] && [ "$FAILED" != "0" ]; then | |
| echo "value=failure" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "value=${{ job.status }}" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Report to CRCR | |
| if: always() | |
| continue-on-error: true | |
| uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main | |
| with: | |
| status: completed | |
| conclusion: ${{ steps.conclusion.outputs.value }} | |
| delivery-id: ${{ needs.build.outputs.sha }} | |
| event-type: nightly | |
| job-name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (cpu, linux.rhel96)" | |
| inductor-tests: | |
| name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (inductor, linux.rhel96)" | |
| needs: [build, determine-tests, cpu-tests] | |
| if: ${{ !cancelled() && needs.build.result == 'success' && needs.determine-tests.result == 'success' && (github.event_name == 'schedule' || inputs.test_categories == 'all' || inputs.test_categories == 'inductor') }} | |
| runs-on: linux.rhel96 | |
| timeout-minutes: 1440 | |
| env: | |
| IMAGE_TAG: ${{ needs.build.outputs.image_tag }} | |
| PYTORCH_SHA: ${{ needs.build.outputs.sha }} | |
| steps: | |
| - name: Verify image exists | |
| run: | | |
| if ! podman image exists "${IMAGE_TAG}"; then | |
| echo "::error::Image ${IMAGE_TAG} not found in local podman store" | |
| exit 1 | |
| fi | |
| - name: Run inductor test suite | |
| id: inductor_run | |
| continue-on-error: true | |
| env: | |
| TEST_LIST_B64: ${{ needs.determine-tests.outputs.inductor_tests }} | |
| run: | | |
| COMMANDS=$(echo "$TEST_LIST_B64" | base64 -d) | |
| if [ -z "$COMMANDS" ]; then | |
| echo "No inductor tests to run" | |
| echo "passed=0" >> "$GITHUB_OUTPUT" | |
| echo "failed=0" >> "$GITHUB_OUTPUT" | |
| echo "errors=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| CMD_COUNT=$(echo "$COMMANDS" | wc -l) | |
| echo "Running ${CMD_COUNT} inductor test command(s)" | |
| LOG=/tmp/inductor_test_output.log | |
| echo "$COMMANDS" > /tmp/inductor_test_commands.txt | |
| podman run --rm \ | |
| -e CONTINUE_THROUGH_ERROR=True \ | |
| -v /tmp/inductor_test_commands.txt:/tmp/test_commands.txt:ro \ | |
| "${IMAGE_TAG}" bash -c ' | |
| source /miniconda/etc/profile.d/conda.sh && conda activate cuda_torch_build | |
| cd /pytorch | |
| PASSED=0; FAILED=0; ERRORS="" | |
| while IFS= read -r cmd; do | |
| [ -z "$cmd" ] && continue | |
| echo "" | |
| echo "============================================================" | |
| echo "Running: $cmd" | |
| echo "============================================================" | |
| echo "$cmd" > /tmp/_run.sh | |
| if timeout 7200 bash /tmp/_run.sh; then | |
| PASSED=$((PASSED + 1)) | |
| else | |
| RC=$? | |
| FAILED=$((FAILED + 1)) | |
| if [ $RC -eq 124 ]; then | |
| ERRORS="${ERRORS}\n TIMEOUT(2h): $cmd" | |
| else | |
| ERRORS="${ERRORS}\n FAIL: $cmd" | |
| fi | |
| fi | |
| done < /tmp/test_commands.txt | |
| echo "" | |
| echo ":::SUMMARY:::" | |
| echo "PASSED=${PASSED}" | |
| echo "FAILED=${FAILED}" | |
| if [ $FAILED -gt 0 ]; then | |
| echo -e "ERRORS=${ERRORS}" | |
| fi | |
| ' 2>&1 | tee "$LOG" || true | |
| PASSED=$(grep '^PASSED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAILED=$(grep '^FAILED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAIL_LIST=$(grep '^\s*\(FAIL\|TIMEOUT\)' "$LOG" || true) | |
| echo "passed=${PASSED:-0}" >> "$GITHUB_OUTPUT" | |
| echo "failed=${FAILED:-0}" >> "$GITHUB_OUTPUT" | |
| { | |
| echo 'errors<<ERRORS_EOF' | |
| echo "$FAIL_LIST" | |
| echo 'ERRORS_EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Test summary | |
| if: always() | |
| env: | |
| ERRORS_OUTPUT: ${{ steps.inductor_run.outputs.errors }} | |
| run: | | |
| echo "## CRCR Nightly: RHEL 9.6 — Inductor Tests" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Nightly SHA | [\`${PYTORCH_SHA}\`](https://github.com/pytorch/pytorch/commit/${PYTORCH_SHA}) |" >> $GITHUB_STEP_SUMMARY | |
| FAILED="${{ steps.inductor_run.outputs.failed || '0' }}" | |
| echo "| Passed | ${{ steps.inductor_run.outputs.passed || '—' }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Failed | ${FAILED} |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${FAILED}" -gt 0 ] 2>/dev/null; then | |
| echo "| Result | \`completed with failures\` |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| Result | \`success\` |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ -n "$ERRORS_OUTPUT" ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Failed Commands" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "$ERRORS_OUTPUT" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Determine conclusion | |
| id: conclusion | |
| if: always() | |
| env: | |
| FAILED: ${{ steps.inductor_run.outputs.failed }} | |
| run: | | |
| if [ -n "$FAILED" ] && [ "$FAILED" != "0" ]; then | |
| echo "value=failure" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "value=${{ job.status }}" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Report to CRCR | |
| if: always() | |
| continue-on-error: true | |
| uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main | |
| with: | |
| status: completed | |
| conclusion: ${{ steps.conclusion.outputs.value }} | |
| delivery-id: ${{ needs.build.outputs.sha }} | |
| event-type: nightly | |
| job-name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (inductor, linux.rhel96)" | |
| sgpu-tests: | |
| name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (sgpu, linux.rhel96)" | |
| needs: [build, determine-tests, inductor-tests] | |
| if: ${{ !cancelled() && needs.build.result == 'success' && needs.determine-tests.result == 'success' && (github.event_name == 'schedule' || inputs.test_categories == 'all' || inputs.test_categories == 'sgpu') }} | |
| runs-on: linux.rhel96 | |
| timeout-minutes: 1440 | |
| env: | |
| IMAGE_TAG: ${{ needs.build.outputs.image_tag }} | |
| PYTORCH_SHA: ${{ needs.build.outputs.sha }} | |
| steps: | |
| - name: Check GPU availability | |
| id: gpu_check | |
| run: | | |
| if nvidia-smi &>/dev/null; then | |
| GPU_COUNT=$(nvidia-smi -L 2>/dev/null | wc -l) | |
| echo "gpu_available=true" >> "$GITHUB_OUTPUT" | |
| echo "gpu_count=$GPU_COUNT" >> "$GITHUB_OUTPUT" | |
| nvidia-smi -L | |
| else | |
| echo "gpu_available=false" >> "$GITHUB_OUTPUT" | |
| echo "gpu_count=0" >> "$GITHUB_OUTPUT" | |
| echo "No GPU detected — skipping single-GPU tests" | |
| fi | |
| - name: Verify image exists | |
| if: steps.gpu_check.outputs.gpu_available == 'true' | |
| run: | | |
| if ! podman image exists "${IMAGE_TAG}"; then | |
| echo "::error::Image ${IMAGE_TAG} not found in local podman store" | |
| exit 1 | |
| fi | |
| - name: Run single-GPU test suite | |
| id: sgpu_run | |
| continue-on-error: true | |
| if: steps.gpu_check.outputs.gpu_available == 'true' | |
| env: | |
| TEST_LIST_B64: ${{ needs.determine-tests.outputs.sgpu_tests }} | |
| run: | | |
| COMMANDS=$(echo "$TEST_LIST_B64" | base64 -d) | |
| if [ -z "$COMMANDS" ]; then | |
| echo "No sgpu tests to run" | |
| echo "passed=0" >> "$GITHUB_OUTPUT" | |
| echo "failed=0" >> "$GITHUB_OUTPUT" | |
| echo "errors=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| CMD_COUNT=$(echo "$COMMANDS" | wc -l) | |
| echo "Running ${CMD_COUNT} single-GPU test command(s)" | |
| LOG=/tmp/sgpu_test_output.log | |
| echo "$COMMANDS" > /tmp/sgpu_test_commands.txt | |
| podman run --rm --device nvidia.com/gpu=all \ | |
| -e CONTINUE_THROUGH_ERROR=True \ | |
| -v /tmp/sgpu_test_commands.txt:/tmp/test_commands.txt:ro \ | |
| "${IMAGE_TAG}" bash -c ' | |
| source /miniconda/etc/profile.d/conda.sh && conda activate cuda_torch_build | |
| cd /pytorch | |
| PASSED=0; FAILED=0; ERRORS="" | |
| while IFS= read -r cmd; do | |
| [ -z "$cmd" ] && continue | |
| echo "" | |
| echo "============================================================" | |
| echo "Running: $cmd" | |
| echo "============================================================" | |
| echo "$cmd" > /tmp/_run.sh | |
| if timeout 7200 bash /tmp/_run.sh; then | |
| PASSED=$((PASSED + 1)) | |
| else | |
| RC=$? | |
| FAILED=$((FAILED + 1)) | |
| if [ $RC -eq 124 ]; then | |
| ERRORS="${ERRORS}\n TIMEOUT(2h): $cmd" | |
| else | |
| ERRORS="${ERRORS}\n FAIL: $cmd" | |
| fi | |
| fi | |
| done < /tmp/test_commands.txt | |
| echo "" | |
| echo ":::SUMMARY:::" | |
| echo "PASSED=${PASSED}" | |
| echo "FAILED=${FAILED}" | |
| if [ $FAILED -gt 0 ]; then | |
| echo -e "ERRORS=${ERRORS}" | |
| fi | |
| ' 2>&1 | tee "$LOG" || true | |
| PASSED=$(grep '^PASSED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAILED=$(grep '^FAILED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAIL_LIST=$(grep '^\s*\(FAIL\|TIMEOUT\)' "$LOG" || true) | |
| echo "passed=${PASSED:-0}" >> "$GITHUB_OUTPUT" | |
| echo "failed=${FAILED:-0}" >> "$GITHUB_OUTPUT" | |
| { | |
| echo 'errors<<ERRORS_EOF' | |
| echo "$FAIL_LIST" | |
| echo 'ERRORS_EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Test summary | |
| if: always() | |
| env: | |
| ERRORS_OUTPUT: ${{ steps.sgpu_run.outputs.errors }} | |
| run: | | |
| echo "## CRCR Nightly: RHEL 9.6 — Single-GPU Tests" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Nightly SHA | [\`${PYTORCH_SHA}\`](https://github.com/pytorch/pytorch/commit/${PYTORCH_SHA}) |" >> $GITHUB_STEP_SUMMARY | |
| echo "| GPUs | ${{ steps.gpu_check.outputs.gpu_count }} |" >> $GITHUB_STEP_SUMMARY | |
| FAILED="${{ steps.sgpu_run.outputs.failed || '0' }}" | |
| echo "| Passed | ${{ steps.sgpu_run.outputs.passed || '—' }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Failed | ${FAILED} |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ steps.gpu_check.outputs.gpu_available }}" != "true" ]; then | |
| echo "| Result | \`skipped (no GPU)\` |" >> $GITHUB_STEP_SUMMARY | |
| elif [ "${FAILED}" -gt 0 ] 2>/dev/null; then | |
| echo "| Result | \`completed with failures\` |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| Result | \`success\` |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ -n "$ERRORS_OUTPUT" ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Failed Commands" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "$ERRORS_OUTPUT" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Determine conclusion | |
| id: conclusion | |
| if: always() | |
| env: | |
| FAILED: ${{ steps.sgpu_run.outputs.failed }} | |
| run: | | |
| if [ -n "$FAILED" ] && [ "$FAILED" != "0" ]; then | |
| echo "value=failure" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "value=${{ job.status }}" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Report to CRCR | |
| if: always() && steps.gpu_check.outputs.gpu_available == 'true' | |
| continue-on-error: true | |
| uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main | |
| with: | |
| status: completed | |
| conclusion: ${{ steps.conclusion.outputs.value }} | |
| delivery-id: ${{ needs.build.outputs.sha }} | |
| event-type: nightly | |
| job-name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (sgpu, linux.rhel96)" | |
| mgpu-tests: | |
| name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (mgpu, linux.rhel96)" | |
| needs: [build, determine-tests, sgpu-tests] | |
| if: ${{ !cancelled() && needs.build.result == 'success' && needs.determine-tests.result == 'success' && (github.event_name == 'schedule' || inputs.test_categories == 'all' || inputs.test_categories == 'mgpu') }} | |
| runs-on: linux.rhel96 | |
| timeout-minutes: 1440 | |
| env: | |
| IMAGE_TAG: ${{ needs.build.outputs.image_tag }} | |
| PYTORCH_SHA: ${{ needs.build.outputs.sha }} | |
| steps: | |
| - name: Check multi-GPU availability | |
| id: gpu_check | |
| run: | | |
| if nvidia-smi &>/dev/null; then | |
| GPU_COUNT=$(nvidia-smi -L 2>/dev/null | wc -l) | |
| echo "gpu_count=$GPU_COUNT" >> "$GITHUB_OUTPUT" | |
| if [ "$GPU_COUNT" -ge 2 ]; then | |
| echo "mgpu_available=true" >> "$GITHUB_OUTPUT" | |
| echo "Found $GPU_COUNT GPUs — multi-GPU tests enabled" | |
| else | |
| echo "mgpu_available=false" >> "$GITHUB_OUTPUT" | |
| echo "Only $GPU_COUNT GPU — skipping multi-GPU tests (need 2+)" | |
| fi | |
| else | |
| echo "mgpu_available=false" >> "$GITHUB_OUTPUT" | |
| echo "gpu_count=0" >> "$GITHUB_OUTPUT" | |
| echo "No GPU detected — skipping multi-GPU tests" | |
| fi | |
| - name: Verify image exists | |
| if: steps.gpu_check.outputs.mgpu_available == 'true' | |
| run: | | |
| if ! podman image exists "${IMAGE_TAG}"; then | |
| echo "::error::Image ${IMAGE_TAG} not found in local podman store" | |
| exit 1 | |
| fi | |
| - name: Run multi-GPU test suite | |
| id: mgpu_run | |
| continue-on-error: true | |
| if: steps.gpu_check.outputs.mgpu_available == 'true' | |
| env: | |
| TEST_LIST_B64: ${{ needs.determine-tests.outputs.mgpu_tests }} | |
| run: | | |
| COMMANDS=$(echo "$TEST_LIST_B64" | base64 -d) | |
| if [ -z "$COMMANDS" ]; then | |
| echo "No mgpu tests to run" | |
| echo "passed=0" >> "$GITHUB_OUTPUT" | |
| echo "failed=0" >> "$GITHUB_OUTPUT" | |
| echo "errors=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| CMD_COUNT=$(echo "$COMMANDS" | wc -l) | |
| echo "Running ${CMD_COUNT} multi-GPU test command(s)" | |
| LOG=/tmp/mgpu_test_output.log | |
| echo "$COMMANDS" > /tmp/mgpu_test_commands.txt | |
| # MGPU-009: cap thread pools, lift pids limit (libgomp thread exhaustion). | |
| podman run --rm --device nvidia.com/gpu=all \ | |
| --pids-limit=-1 \ | |
| -e CONTINUE_THROUGH_ERROR=True \ | |
| -e OMP_NUM_THREADS=8 \ | |
| -e MKL_NUM_THREADS=8 \ | |
| -v /tmp/mgpu_test_commands.txt:/tmp/test_commands.txt:ro \ | |
| "${IMAGE_TAG}" bash -c ' | |
| source /miniconda/etc/profile.d/conda.sh && conda activate cuda_torch_build | |
| cd /pytorch | |
| PASSED=0; FAILED=0; ERRORS="" | |
| while IFS= read -r cmd; do | |
| [ -z "$cmd" ] && continue | |
| echo "" | |
| echo "============================================================" | |
| echo "Running: $cmd" | |
| echo "============================================================" | |
| echo "$cmd" > /tmp/_run.sh | |
| if timeout 43200 bash /tmp/_run.sh; then | |
| PASSED=$((PASSED + 1)) | |
| else | |
| RC=$? | |
| FAILED=$((FAILED + 1)) | |
| if [ $RC -eq 124 ]; then | |
| ERRORS="${ERRORS}\n TIMEOUT(12h): $cmd" | |
| else | |
| ERRORS="${ERRORS}\n FAIL: $cmd" | |
| fi | |
| fi | |
| done < /tmp/test_commands.txt | |
| echo "" | |
| echo ":::SUMMARY:::" | |
| echo "PASSED=${PASSED}" | |
| echo "FAILED=${FAILED}" | |
| if [ $FAILED -gt 0 ]; then | |
| echo -e "ERRORS=${ERRORS}" | |
| fi | |
| ' 2>&1 | tee "$LOG" || true | |
| PASSED=$(grep '^PASSED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAILED=$(grep '^FAILED=' "$LOG" | tail -1 | cut -d= -f2) | |
| FAIL_LIST=$(grep '^\s*\(FAIL\|TIMEOUT\)' "$LOG" || true) | |
| echo "passed=${PASSED:-0}" >> "$GITHUB_OUTPUT" | |
| echo "failed=${FAILED:-0}" >> "$GITHUB_OUTPUT" | |
| { | |
| echo 'errors<<ERRORS_EOF' | |
| echo "$FAIL_LIST" | |
| echo 'ERRORS_EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Test summary | |
| if: always() | |
| env: | |
| ERRORS_OUTPUT: ${{ steps.mgpu_run.outputs.errors }} | |
| run: | | |
| echo "## CRCR Nightly: RHEL 9.6 — Multi-GPU Tests" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Nightly SHA | [\`${PYTORCH_SHA}\`](https://github.com/pytorch/pytorch/commit/${PYTORCH_SHA}) |" >> $GITHUB_STEP_SUMMARY | |
| echo "| GPUs | ${{ steps.gpu_check.outputs.gpu_count }} |" >> $GITHUB_STEP_SUMMARY | |
| FAILED="${{ steps.mgpu_run.outputs.failed || '0' }}" | |
| echo "| Passed | ${{ steps.mgpu_run.outputs.passed || '—' }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Failed | ${FAILED} |" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ steps.gpu_check.outputs.mgpu_available }}" != "true" ]; then | |
| echo "| Result | \`skipped (need 2+ GPUs)\` |" >> $GITHUB_STEP_SUMMARY | |
| elif [ "${FAILED}" -gt 0 ] 2>/dev/null; then | |
| echo "| Result | \`completed with failures\` |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| Result | \`success\` |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ -n "$ERRORS_OUTPUT" ]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Failed Commands" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "$ERRORS_OUTPUT" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Determine conclusion | |
| id: conclusion | |
| if: always() | |
| env: | |
| FAILED: ${{ steps.mgpu_run.outputs.failed }} | |
| run: | | |
| if [ -n "$FAILED" ] && [ "$FAILED" != "0" ]; then | |
| echo "value=failure" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "value=${{ job.status }}" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Report to CRCR | |
| if: always() && steps.gpu_check.outputs.mgpu_available == 'true' | |
| continue-on-error: true | |
| uses: pytorch/test-infra/.github/actions/cross-repo-ci-relay-callback@main | |
| with: | |
| status: completed | |
| conclusion: ${{ steps.conclusion.outputs.value }} | |
| delivery-id: ${{ needs.build.outputs.sha }} | |
| event-type: nightly | |
| job-name: "linux-rhel9.6-cuda13.0-py3.12-gcc11-x86_64 / test (mgpu, linux.rhel96)" |