Skip to content

Commit a6b9b70

Browse files
aryangorwadearyangorwade
committed
Added CI test
Signed-off-by: Aryan Gorwade <gorwadearyan@gmail.com>
1 parent 11c23d0 commit a6b9b70

2 files changed

Lines changed: 216 additions & 0 deletions

File tree

‎.github/workflows/e2e-tests.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,17 @@ jobs:
6565
operator_image: ${{ inputs.operator_image }}
6666
operator_version: ${{ inputs.operator_version }}
6767

68+
ha-failover:
69+
needs: [variables]
70+
if: needs.variables.outputs.push_on_build == 'true'
71+
permissions:
72+
contents: read
73+
packages: read
74+
uses: ./.github/workflows/ha-failover.yaml
75+
with:
76+
operator_image: ${{ needs.variables.outputs.operator_image }}
77+
operator_version: ${{ needs.variables.outputs.operator_version }}
78+
6879
publish-helm-oci-chart:
6980
needs: [variables]
7081
permissions:

‎.github/workflows/ha-failover.yaml‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
# Copyright NVIDIA CORPORATION
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: HA Failover
16+
17+
on:
18+
workflow_call:
19+
inputs:
20+
operator_image:
21+
required: true
22+
type: string
23+
operator_version:
24+
required: true
25+
type: string
26+
27+
permissions:
28+
contents: read
29+
packages: read
30+
31+
jobs:
32+
ha-failover:
33+
runs-on: ubuntu-24.04
34+
timeout-minutes: 15
35+
defaults:
36+
run:
37+
shell: bash
38+
env:
39+
KUBECONFIG: /tmp/ha-kubeconfig
40+
OPERATOR_IMAGE: ${{ inputs.operator_image }}
41+
OPERATOR_VERSION: ${{ inputs.operator_version }}
42+
steps:
43+
- name: Check out code
44+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
45+
with:
46+
persist-credentials: false
47+
- name: Login to GitHub Container Registry
48+
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
49+
with:
50+
registry: ghcr.io
51+
username: ${{ github.actor }}
52+
password: ${{ secrets.GITHUB_TOKEN }}
53+
- name: Set up Helm
54+
uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
55+
with:
56+
version: v3.19.0
57+
- name: Create kind cluster and install KWOK
58+
timeout-minutes: 5
59+
run: |
60+
mkdir -p "$RUNNER_TEMP/ha-bin"
61+
curl --fail --location --retry 3 https://kind.sigs.k8s.io/dl/v0.33.0/kind-linux-amd64 \
62+
-o "$RUNNER_TEMP/ha-bin/kind"
63+
chmod +x "$RUNNER_TEMP/ha-bin/kind"
64+
export PATH="$RUNNER_TEMP/ha-bin:$PATH"
65+
echo "$RUNNER_TEMP/ha-bin" >> "$GITHUB_PATH"
66+
cat > "$RUNNER_TEMP/ha-kind.yaml" <<'YAML'
67+
kind: Cluster
68+
apiVersion: kind.x-k8s.io/v1alpha4
69+
nodes:
70+
- role: control-plane
71+
YAML
72+
kind create cluster --name gpu-operator-ha --config "$RUNNER_TEMP/ha-kind.yaml" \
73+
--image kindest/node@sha256:a1ed56cfb0e7b93589bdf97c8cd566405a265939e3620fc4f5de89adff580ae5 --wait 120s
74+
docker cp gpu-operator-ha-control-plane:/usr/bin/kubectl "$RUNNER_TEMP/ha-bin/kubectl"
75+
kubectl label nodes --all ha-test.nvidia.com/real-node=true
76+
# These DaemonSets tolerate all taints; keep them off simulated nodes.
77+
for daemonset in kindnet kube-proxy; do
78+
kubectl -n kube-system patch daemonset "$daemonset" --type=merge \
79+
-p '{"spec":{"template":{"spec":{"nodeSelector":{"ha-test.nvidia.com/real-node":"true"}}}}}'
80+
done
81+
kwok_url=https://github.com/kubernetes-sigs/kwok/releases/download/v0.8.0
82+
kubectl apply -f "$kwok_url/kwok.yaml"
83+
kubectl wait --for=condition=Established crd/stages.kwok.x-k8s.io --timeout=60s
84+
kubectl apply -f "$kwok_url/stage-fast.yaml"
85+
kubectl -n kube-system patch deployment kwok-controller --type=json -p '[
86+
{"op":"add","path":"/spec/template/spec/nodeSelector","value":{"node-role.kubernetes.io/control-plane":""}},
87+
{"op":"add","path":"/spec/template/spec/tolerations","value":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists","effect":"NoSchedule"}]},
88+
{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--v=DEBUG"}
89+
]'
90+
kubectl -n kube-system rollout status deployment/kwok-controller --timeout=90s
91+
kubectl get node gpu-operator-ha-control-plane -o json | jq '{
92+
apiVersion: "v1", kind: "Node",
93+
metadata: {name: "kwok-gpu", annotations: {"kwok.x-k8s.io/node": "fake"}, labels: {
94+
"kubernetes.io/hostname": "kwok-gpu", "kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64",
95+
"feature.node.kubernetes.io/pci-10de.present": "true",
96+
"feature.node.kubernetes.io/system-os_release.ID": "ubuntu",
97+
"feature.node.kubernetes.io/system-os_release.VERSION_ID": "24.04",
98+
"feature.node.kubernetes.io/kernel-version.full": "6.8.0-60-generic"}},
99+
spec: {taints: [{key: "kwok.x-k8s.io/node", value: "fake", effect: "NoSchedule"}]},
100+
status: {nodeInfo: .status.nodeInfo,
101+
capacity: (.status.capacity + {"nvidia.com/gpu": "1"}),
102+
allocatable: (.status.allocatable + {"nvidia.com/gpu": "1"})}
103+
}' | kubectl apply -f -
104+
kubectl wait --for=condition=Ready node/kwok-gpu --timeout=60s
105+
- name: Install two operator replicas
106+
timeout-minutes: 5
107+
run: |
108+
docker pull "$OPERATOR_IMAGE:$OPERATOR_VERSION"
109+
kind load docker-image "$OPERATOR_IMAGE:$OPERATOR_VERSION" --name gpu-operator-ha
110+
cat > "$RUNNER_TEMP/ha-values.yaml" <<'YAML'
111+
nfd:
112+
enabled: false
113+
operator:
114+
replicaCount: 2
115+
imagePullPolicy: Never
116+
leaderElection:
117+
warmup: false
118+
nodeSelector:
119+
kubernetes.io/hostname: gpu-operator-ha-control-plane
120+
daemonsets:
121+
tolerations:
122+
- key: kwok.x-k8s.io/node
123+
operator: Equal
124+
value: fake
125+
effect: NoSchedule
126+
YAML
127+
helm install gpu-operator ./deployments/gpu-operator --namespace gpu-operator --create-namespace \
128+
-f "$RUNNER_TEMP/ha-values.yaml" --set operator.repository="${OPERATOR_IMAGE%/*}" \
129+
--set operator.image="${OPERATOR_IMAGE##*/}" --set operator.version="$OPERATOR_VERSION" \
130+
--wait --timeout 180s
131+
- name: Kill leader and verify takeover and reconciliation
132+
timeout-minutes: 3
133+
run: |
134+
wait_for() {
135+
local deadline=$((SECONDS + $1))
136+
shift
137+
until "$@"; do
138+
(( SECONDS < deadline )) || { echo "::error::Timed out: $*"; return 1; }
139+
sleep 1
140+
done
141+
}
142+
lease_holder() {
143+
kubectl -n gpu-operator get lease 53822513.nvidia.com -o jsonpath='{.spec.holderIdentity}'
144+
}
145+
operands_ready() {
146+
kubectl -n gpu-operator get daemonsets -o json | jq -e '
147+
.items | length > 0 and any(.[]; .status.desiredNumberScheduled == 1) and all(.[];
148+
.status.observedGeneration == .metadata.generation and
149+
(.status.numberReady // 0) == .status.desiredNumberScheduled)' > /dev/null
150+
}
151+
wait_for 90 operands_ready
152+
kubectl -n gpu-operator get pods -l app=gpu-operator -o json > "$RUNNER_TEMP/ha-pods.json"
153+
jq -e '.items | length == 2' "$RUNNER_TEMP/ha-pods.json"
154+
holder=$(lease_holder)
155+
leader=${holder%%_*}
156+
node=$(jq -r --arg pod "$leader" '.items[] | select(.metadata.name == $pod) | .spec.nodeName' "$RUNNER_TEMP/ha-pods.json")
157+
container=$(kubectl -n gpu-operator get pod "$leader" -o jsonpath='{.status.containerStatuses[0].containerID}')
158+
pid=$(docker exec "$node" crictl inspect "${container#containerd://}" | jq -r '.info.pid')
159+
[[ "$pid" =~ ^[1-9][0-9]*$ ]]
160+
# Keep the killed replica from restarting and winning the Lease again.
161+
trap 'docker exec "$node" systemctl start kubelet' EXIT
162+
docker exec "$node" systemctl stop kubelet
163+
[[ $(lease_holder) == "$holder" ]]
164+
t0=$SECONDS
165+
docker exec "$node" kill -KILL "$pid"
166+
acquired() {
167+
successor=$(lease_holder)
168+
successor=${successor%%_*}
169+
[[ "$successor" != "$leader" ]] && jq -e --arg pod "$successor" \
170+
'any(.items[]; .metadata.name == $pod)' "$RUNNER_TEMP/ha-pods.json" > /dev/null
171+
}
172+
wait_for 60 acquired
173+
marker="ha-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
174+
kubectl patch clusterpolicy cluster-policy --type=merge \
175+
-p "{\"spec\":{\"daemonsets\":{\"annotations\":{\"ha-test.nvidia.com/failover\":\"$marker\"}}}}"
176+
reconciled() {
177+
kubectl -n gpu-operator get daemonsets -o json | jq -e --arg marker "$marker" '
178+
.items | length > 0 and all(.[]; .spec.template.metadata.annotations["ha-test.nvidia.com/failover"] == $marker)' > /dev/null
179+
}
180+
wait_for 60 reconciled
181+
[[ $(lease_holder) == "${successor}_"* ]]
182+
echo "Leader $leader killed; standby $successor reconciled in $((SECONDS - t0))s"
183+
(( SECONDS - t0 <= 60 ))
184+
- name: Show failure details
185+
if: failure()
186+
run: |
187+
kubectl --request-timeout=10s -n gpu-operator get pods,daemonsets,leases -o wide || true
188+
kubectl --request-timeout=10s get node kwok-gpu -o yaml || true
189+
kubectl --request-timeout=10s -n kube-node-lease get lease kwok-gpu -o yaml || true
190+
kubectl --request-timeout=10s -n gpu-operator logs -l app=gpu-operator --prefix --tail=100 || true
191+
kubectl --request-timeout=10s -n kube-system describe pods -l app=kwok-controller || true
192+
kubectl --request-timeout=10s -n kube-system logs -l app=kwok-controller --prefix --tail=-1 || true
193+
if kubectl --request-timeout=10s -n kube-system get pods -l app=kwok-controller -o json |
194+
jq -e 'any(.items[].status.containerStatuses[]?; .restartCount > 0)' > /dev/null; then
195+
kubectl --request-timeout=10s -n kube-system logs -l app=kwok-controller --previous --prefix --tail=-1 || true
196+
fi
197+
pending=$(kubectl --request-timeout=10s -n gpu-operator get pods \
198+
--field-selector=spec.nodeName=kwok-gpu,status.phase=Pending -o json |
199+
jq -r '.items[0].metadata.name // empty') || true
200+
if [[ -n "$pending" ]]; then
201+
kubectl --request-timeout=10s -n gpu-operator describe pod "$pending" || true
202+
fi
203+
- name: Delete kind cluster
204+
if: always()
205+
run: kind delete cluster --name gpu-operator-ha

0 commit comments

Comments
 (0)