From 5342e8d799cd7f38d19d770fd0506a7e9fde2c60 Mon Sep 17 00:00:00 2001 From: Abrar Shivani Date: Thu, 20 Aug 2026 21:59:38 -0700 Subject: [PATCH] install the published helm oci chart in e2e tests The e2e workflow already packages and pushes the chart as an OCI artifact and gates both test jobs on that job succeeding, but the tests then installed the chart directory straight from the checkout. CI never exercised the artifact we publish. Expose the chart reference and version as outputs of the publish workflow and hand them to the test scripts as HELM_CHART and HELM_CHART_VERSION. When HELM_CHART is empty the scripts fall back to the chart in the working tree, so running tests/local.sh by hand still tests local changes. The default has to stay empty rather than an absolute path: local.sh forwards the variable over ssh and the controller's PROJECT_DIR does not exist on the test instance. Both e2e jobs now run helm show chart before setting up holodeck, so a missing or unpropagated tag fails in seconds instead of after an instance has been provisioned. Co-authored-by: Tariq Signed-off-by: Abrar Shivani --- .github/workflows/e2e-tests.yaml | 18 ++++++++++++++++++ .github/workflows/publish-helm-oci-chart.yaml | 13 +++++++++++++ tests/local.sh | 2 ++ tests/scripts/.definitions.sh | 6 ++++++ tests/scripts/end-to-end-nvidia-driver.sh | 6 +++--- tests/scripts/install-operator.sh | 4 ++-- .../migrate-clusterpolicy-to-nvidiadriver.sh | 2 +- 7 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 94d929902b..1d3eaaedf5 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -90,6 +90,13 @@ jobs: with: name: values-overrides path: ${{ github.workspace }} + - name: Set up Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Verify the published Helm chart is available + env: + HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} + run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}" - name: Set up Holodeck uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7 with: @@ -124,6 +131,8 @@ jobs: env: OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} GPU_PRODUCT_NAME: "Tesla-T4" SKIP_LAUNCH: "true" CONTAINER_RUNTIME: "containerd" @@ -158,6 +167,13 @@ jobs: with: name: values-overrides path: ${{ github.workspace }} + - name: Set up Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Verify the published Helm chart is available + env: + HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} + run: helm show chart "${HELM_CHART}" --version "${HELM_CHART_VERSION}" - name: Set up Holodeck uses: NVIDIA/holodeck@e0f3932cf284d92421a55536839fa821a14116aa # v0.3.7 with: @@ -192,6 +208,8 @@ jobs: env: OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }} OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }} + HELM_CHART: ${{ needs.publish-helm-oci-chart.outputs.chart_reference }} + HELM_CHART_VERSION: ${{ needs.publish-helm-oci-chart.outputs.chart_version }} GPU_PRODUCT_NAME: "Tesla-T4" SKIP_LAUNCH: "true" CONTAINER_RUNTIME: "containerd" diff --git a/.github/workflows/publish-helm-oci-chart.yaml b/.github/workflows/publish-helm-oci-chart.yaml index 6d56f6d3df..edb01ef11c 100644 --- a/.github/workflows/publish-helm-oci-chart.yaml +++ b/.github/workflows/publish-helm-oci-chart.yaml @@ -23,6 +23,13 @@ on: operator_version: required: true type: string + outputs: + chart_reference: + description: "The OCI reference of the published Helm chart" + value: ${{ jobs.publish-helm-chart.outputs.chart_reference }} + chart_version: + description: "The version of the published Helm chart" + value: ${{ jobs.publish-helm-chart.outputs.chart_version }} permissions: contents: read @@ -36,6 +43,9 @@ jobs: publish-helm-chart: runs-on: ubuntu-latest timeout-minutes: 10 + outputs: + chart_reference: ${{ steps.publish.outputs.chart_reference }} + chart_version: ${{ steps.package.outputs.chart_version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 name: Check out code @@ -89,6 +99,7 @@ jobs: echo "chart_version=${CHART_VERSION}" >> "${GITHUB_OUTPUT}" - name: Publish Helm OCI chart + id: publish env: GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} CHART_PACKAGE: ${{ steps.package.outputs.chart_package }} @@ -100,4 +111,6 @@ jobs: CHART_REPOSITORY="oci://ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator/charts" helm push "${CHART_PACKAGE}" "${CHART_REPOSITORY}" + + echo "chart_reference=${CHART_REPOSITORY}/gpu-operator" >> "${GITHUB_OUTPUT}" echo "::notice::Published Helm chart to ${CHART_REPOSITORY}/gpu-operator:${CHART_VERSION}" diff --git a/tests/local.sh b/tests/local.sh index 33a721d782..9981de85f7 100755 --- a/tests/local.sh +++ b/tests/local.sh @@ -42,4 +42,6 @@ remote \ GPU_MODE="${GPU_MODE}" \ NGC_API_KEY="${NGC_API_KEY}" \ VALUES_FILE="${VALUES_FILE:-}" \ + HELM_CHART="${HELM_CHART:-}" \ + HELM_CHART_VERSION="${HELM_CHART_VERSION:-}" \ ${TEST_CASE} diff --git a/tests/scripts/.definitions.sh b/tests/scripts/.definitions.sh index 42ed034f1d..737a019c6e 100644 --- a/tests/scripts/.definitions.sh +++ b/tests/scripts/.definitions.sh @@ -26,3 +26,9 @@ TERRAFORM="terraform -chdir=${TERRAFORM_DIR}" : ${PRIVATE_REGISTRY:="nvcr.io"} : ${GPU_MODE:="gpu"} + +CHART_REFERENCE="${HELM_CHART:-${PROJECT_DIR}/deployments/gpu-operator}" +HELM_CMD_ARGS=("${CHART_REFERENCE}") +if [[ -n "${HELM_CHART_VERSION:-}" ]]; then + HELM_CMD_ARGS+=(--version "${HELM_CHART_VERSION}") +fi diff --git a/tests/scripts/end-to-end-nvidia-driver.sh b/tests/scripts/end-to-end-nvidia-driver.sh index cd5408645b..b593ad309f 100755 --- a/tests/scripts/end-to-end-nvidia-driver.sh +++ b/tests/scripts/end-to-end-nvidia-driver.sh @@ -7,14 +7,14 @@ test_nvidiadriver_helm_render_options() { local render_file render_file=$(mktemp) - ${HELM} template gpu-operator "${PROJECT_DIR}/deployments/gpu-operator" \ + ${HELM} template gpu-operator "${HELM_CMD_ARGS[@]}" \ -n "${TEST_NAMESPACE}" \ --set driver.nvidiaDriverCRD.enabled=true \ --set driver.nvidiaDriverCRD.deployDefaultCR=true > "${render_file}" grep -q "kind: NVIDIADriver" "${render_file}" grep -q "default: true" "${render_file}" - ${HELM} template gpu-operator "${PROJECT_DIR}/deployments/gpu-operator" \ + ${HELM} template gpu-operator "${HELM_CMD_ARGS[@]}" \ -n "${TEST_NAMESPACE}" \ --set driver.nvidiaDriverCRD.enabled=true \ --set driver.nvidiaDriverCRD.deployDefaultCR=false > "${render_file}" @@ -23,7 +23,7 @@ test_nvidiadriver_helm_render_options() { exit 1 fi - ${HELM} template gpu-operator "${PROJECT_DIR}/deployments/gpu-operator" \ + ${HELM} template gpu-operator "${HELM_CMD_ARGS[@]}" \ -n "${TEST_NAMESPACE}" \ --set driver.nvidiaDriverCRD.enabled=false \ --set driver.nvidiaDriverCRD.deployDefaultCR=true > "${render_file}" diff --git a/tests/scripts/install-operator.sh b/tests/scripts/install-operator.sh index d874519b1b..71180c6535 100755 --- a/tests/scripts/install-operator.sh +++ b/tests/scripts/install-operator.sh @@ -105,7 +105,7 @@ echo "Operator image: ${OPERATOR_IMAGE}:${OPERATOR_VERSION}" if [[ "${USE_VALUES_FILE}" == "true" ]]; then echo "Using values file approach: ${VALUES_FILE}" - ${HELM} install ${PROJECT_DIR}/deployments/gpu-operator --generate-name \ + ${HELM} install "${HELM_CMD_ARGS[@]}" --generate-name \ -n "${TEST_NAMESPACE}" \ ${EXTRA_VALUES_FILES} \ -f "${VALUES_FILE}" \ @@ -113,7 +113,7 @@ if [[ "${USE_VALUES_FILE}" == "true" ]]; then --wait else echo "Using --set flags approach" - ${HELM} install ${PROJECT_DIR}/deployments/gpu-operator --generate-name \ + ${HELM} install "${HELM_CMD_ARGS[@]}" --generate-name \ -n "${TEST_NAMESPACE}" \ ${OPERATOR_OPTIONS} \ ${TOOLKIT_CONTAINER_OPTIONS} \ diff --git a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh index 8cbaae72c5..4f56d9f198 100755 --- a/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh +++ b/tests/scripts/migrate-clusterpolicy-to-nvidiadriver.sh @@ -174,7 +174,7 @@ if [[ -z "${operator_name}" ]]; then fi echo "Migrating Helm release/${operator_name} from ClusterPolicy driver management to NVIDIADriver" -${HELM} upgrade "${operator_name}" "${PROJECT_DIR}/deployments/gpu-operator" \ +${HELM} upgrade "${operator_name}" "${HELM_CMD_ARGS[@]}" \ -n "${TEST_NAMESPACE}" \ --reuse-values \ ${OPERATOR_OPTIONS:-} \