Repository navigation
perf(async-worker): keep n messages in flight instead of draining a b… #205
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release images | |
| # Builds and pushes the three OCI images that the Helm chart consumes: | |
| # - web (Next.js) | |
| # - edge-functions (Deno runtime + supabase/functions/) | |
| # - migrations (postgres-client + supabase/migrations/) | |
| # and publishes the chart itself as an OCI artifact. | |
| # | |
| # Triggers: | |
| # - push to main → tag `main-<sha>` + `main-latest` (no chart push) | |
| # - push to other branch → tag `<branch>-<sha>` + `<branch>-latest` (no chart push) | |
| # - tag v* → tag <version> + `latest`, plus chart push | |
| # - workflow_dispatch → manual, with optional hostname/version overrides | |
| on: | |
| push: | |
| branches: [main, staging] | |
| tags: ["v*"] | |
| workflow_dispatch: | |
| inputs: | |
| web_hostname: | |
| description: "NEXT_PUBLIC_PAWTOGRADER_WEB_URL build arg (full https URL)" | |
| required: false | |
| default: "https://staging.pawtograder.net" | |
| api_hostname: | |
| description: "NEXT_PUBLIC_SUPABASE_URL build arg (full https URL)" | |
| required: false | |
| default: "https://api.staging.pawtograder.net" | |
| namespace: | |
| description: "Namespace whose pawtograder-jwt:ANON_KEY is baked into the web bundle" | |
| required: false | |
| default: "pawtograder-staging" | |
| channel: | |
| description: "Deployment channel name (A/B). Empty/stable = primary build; a name like 'team7' bakes NEXT_PUBLIC_PAWTOGRADER_CHANNEL and tags the image '<channel>-<sha>'. Set web_hostname to the channel's own host." | |
| required: false | |
| default: "" | |
| channel_host_suffix: | |
| description: "NEXT_PUBLIC_CHANNEL_HOST_SUFFIX build arg (the channel zone, e.g. 'staging.pawtograder.net') — enables per-course host redirect + cross-channel cookie scope. Empty falls back to the vars.CHANNEL_HOST_SUFFIX repo variable; routing is disabled only when both are empty." | |
| required: false | |
| default: "" | |
| permissions: | |
| contents: read | |
| packages: write | |
| env: | |
| REGISTRY: ghcr.io | |
| REPO_OWNER: ${{ github.repository_owner }} | |
| # Silence the Node20 deprecation warnings on docker/* actions that don't | |
| # yet ship a Node24-tagged major; opts in to the runner's Node24 binary | |
| # for every JS-based action in this workflow. | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" | |
| concurrency: | |
| group: release-images-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| meta: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.v.outputs.version }} | |
| is_tag: ${{ steps.v.outputs.is_tag }} | |
| tags: ${{ steps.v.outputs.tags }} | |
| chart_version: ${{ steps.v.outputs.chart_version }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - id: v | |
| name: Compute version + tag list | |
| env: | |
| CHANNEL_INPUT: ${{ inputs.channel }} | |
| run: | | |
| set -euo pipefail | |
| chart_version=$(awk '/^version:/ {print $2; exit}' charts/pawtograder/Chart.yaml) | |
| short_sha=${GITHUB_SHA:0:7} | |
| if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then | |
| version="${GITHUB_REF#refs/tags/v}" | |
| is_tag=true | |
| tags="${version},latest,sha-${short_sha}" | |
| elif [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then | |
| version="main-${short_sha}" | |
| is_tag=false | |
| tags="main-${short_sha},main-latest" | |
| else | |
| # Any other branch added to on.push.branches above. Sanitize `/` | |
| # to `-` so feature branches don't break docker tag syntax. | |
| branch="${GITHUB_REF#refs/heads/}" | |
| safe_branch="${branch//\//-}" | |
| version="${safe_branch}-${short_sha}" | |
| is_tag=false | |
| tags="${safe_branch}-${short_sha},${safe_branch}-latest" | |
| fi | |
| # A/B channel build (workflow_dispatch only): tag the image by channel | |
| # so it never collides with the stable tag, and so the chart can pin | |
| # channels[].web.image.tag to "<channel>-<sha>" / "<channel>-latest". | |
| channel="${CHANNEL_INPUT:-}" | |
| if [[ -n "$channel" && "$channel" != "stable" ]]; then | |
| if [[ ! "$channel" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; then | |
| echo "::error::channel '${channel}' must be a DNS-1123 label (lowercase alphanumeric + '-', <=63 chars)"; exit 1 | |
| fi | |
| version="${channel}-${short_sha}" | |
| tags="${channel}-${short_sha},${channel}-latest" | |
| is_tag=false | |
| fi | |
| echo "version=${version}" | tee -a "$GITHUB_OUTPUT" | |
| echo "is_tag=${is_tag}" | tee -a "$GITHUB_OUTPUT" | |
| echo "tags=${tags}" | tee -a "$GITHUB_OUTPUT" | |
| echo "chart_version=${chart_version}" | tee -a "$GITHUB_OUTPUT" | |
| build-web: | |
| needs: meta | |
| runs-on: pawtograder-ci | |
| # NEXT_PUBLIC_* values are baked into the client bundle by Next.js, so | |
| # one image cannot serve arbitrary deployments. We only build web on | |
| # events where the bake target is unambiguous: | |
| # - workflow_dispatch (caller supplies inputs.web_hostname / | |
| # inputs.api_hostname / inputs.namespace; | |
| # defaults to staging) | |
| # - push to refs/heads/main (staging — main-latest is contractually | |
| # a staging-baked image) | |
| # For tag releases (v*) and other branches, the caller must | |
| # invoke workflow_dispatch with explicit URLs; otherwise we'd silently | |
| # bake staging URLs into a release. edge-functions / migrations / chart | |
| # don't have this problem and continue to build/publish. | |
| # | |
| # NEXT_PUBLIC_SUPABASE_ANON_KEY is read live from the target cluster's | |
| # `<namespace>/pawtograder-jwt` Secret (see "Read anon key from cluster" | |
| # below), NOT from a static repo secret. This is the same approach | |
| # preview.yml uses, and it's deliberate: the anon key is derived from the | |
| # cluster's JWT_SECRET, so a static copy drifts the moment the bundle is | |
| # rotated and silently breaks browser login (gateway 401). Reading it from | |
| # the cluster at build time guarantees build == runtime. | |
| if: github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: azure/setup-kubectl@v4 | |
| with: | |
| version: v1.32.0 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Resolve hostnames + namespace | |
| id: hosts | |
| # CHANNEL / SUFFIX bake the A/B-channel routing into the bundle (see | |
| # utils/channels.ts). On a channel build (workflow_dispatch with a | |
| # `channel` input) the caller supplies the channel's own web_hostname and | |
| # channel_host_suffix. On stable push/dispatch the suffix comes from the | |
| # CHANNEL_HOST_SUFFIX repo variable (empty by default => routing disabled, | |
| # so stable behaves exactly as before until an operator opts in fleet-wide | |
| # by setting that variable). | |
| env: | |
| EVENT: ${{ github.event_name }} | |
| INPUT_WEB: ${{ inputs.web_hostname }} | |
| INPUT_API: ${{ inputs.api_hostname }} | |
| INPUT_NS: ${{ inputs.namespace }} | |
| INPUT_CHANNEL: ${{ inputs.channel }} | |
| INPUT_SUFFIX: ${{ inputs.channel_host_suffix }} | |
| VAR_SUFFIX: ${{ vars.CHANNEL_HOST_SUFFIX }} | |
| run: | | |
| set -euo pipefail | |
| if [ "$EVENT" = "workflow_dispatch" ]; then | |
| web="${INPUT_WEB:-https://staging.pawtograder.net}" | |
| api="${INPUT_API:-https://api.staging.pawtograder.net}" | |
| ns="${INPUT_NS:-pawtograder-staging}" | |
| channel="${INPUT_CHANNEL:-stable}" | |
| suffix="${INPUT_SUFFIX:-${VAR_SUFFIX:-}}" | |
| else | |
| # push to main → staging (see job-level if condition above). | |
| web="https://staging.pawtograder.net" | |
| api="https://api.staging.pawtograder.net" | |
| ns="pawtograder-staging" | |
| channel="stable" | |
| suffix="${VAR_SUFFIX:-}" | |
| fi | |
| # Normalize an empty channel to "stable" (the build's default). | |
| [ -n "$channel" ] || channel="stable" | |
| echo "web=$web" >> "$GITHUB_OUTPUT" | |
| echo "api=$api" >> "$GITHUB_OUTPUT" | |
| echo "ns=$ns" >> "$GITHUB_OUTPUT" | |
| echo "channel=$channel" >> "$GITHUB_OUTPUT" | |
| echo "suffix=$suffix" >> "$GITHUB_OUTPUT" | |
| - name: Read anon key from cluster | |
| # The anon key the browser ships MUST match what the cluster's GoTrue / | |
| # PostgREST validate against, or login fails with a gateway 401. Read it | |
| # straight from the target namespace's pawtograder-jwt Secret so the | |
| # baked value can never drift from the running JWT bundle. Mirrors the | |
| # preview.yml build-web step. | |
| env: | |
| KUBECONFIG_B64: ${{ secrets.KUBECONFIG_BASE64 }} | |
| NS: ${{ steps.hosts.outputs.ns }} | |
| run: | | |
| set -euo pipefail | |
| [ -n "$KUBECONFIG_B64" ] || { echo "::error::KUBECONFIG_BASE64 secret is required to read the anon key from the cluster"; exit 1; } | |
| mkdir -p "$HOME/.kube" | |
| echo "$KUBECONFIG_B64" | base64 -d > "$HOME/.kube/config" | |
| chmod 600 "$HOME/.kube/config" | |
| ANON=$(kubectl -n "$NS" get secret pawtograder-jwt -o jsonpath='{.data.ANON_KEY}' | base64 -d) | |
| [ -n "$ANON" ] || { echo "::error::ANON_KEY missing in $NS/pawtograder-jwt"; exit 1; } | |
| echo "::add-mask::$ANON" | |
| echo "ANON_KEY=$ANON" >> "$GITHUB_ENV" | |
| - name: Build & push web image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| push: true | |
| tags: ${{ format('{0}/{1}/web:{2}', env.REGISTRY, env.REPO_OWNER, needs.meta.outputs.version) }} | |
| build-args: | | |
| NEXT_PUBLIC_PAWTOGRADER_WEB_URL=${{ steps.hosts.outputs.web }} | |
| NEXT_PUBLIC_SUPABASE_URL=${{ steps.hosts.outputs.api }} | |
| NEXT_PUBLIC_SUPABASE_ANON_KEY=${{ env.ANON_KEY }} | |
| NEXT_PUBLIC_GITHUB_OAUTH_CLIENT_ID=${{ secrets.STAGING_GITHUB_OAUTH_CLIENT_ID }} | |
| NEXT_PUBLIC_BUGSINK_DSN=${{ secrets.STAGING_BUGSINK_DSN }} | |
| NEXT_PUBLIC_BUGSINK_HOST=${{ secrets.STAGING_BUGSINK_HOST }} | |
| NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.STAGING_POSTHOG_KEY }} | |
| NEXT_PUBLIC_POSTHOG_HOST=${{ secrets.STAGING_POSTHOG_HOST }} | |
| NEXT_PUBLIC_POSTHOG_UI_HOST=${{ secrets.STAGING_POSTHOG_UI_HOST }} | |
| NEXT_PUBLIC_ENABLE_SIGNUPS=false | |
| NEXT_PUBLIC_GIT_COMMIT_SHA=${{ github.sha }} | |
| NEXT_PUBLIC_PAWTOGRADER_CHANNEL=${{ steps.hosts.outputs.channel }} | |
| NEXT_PUBLIC_CHANNEL_HOST_SUFFIX=${{ steps.hosts.outputs.suffix }} | |
| SENTRY_RELEASE=${{ needs.meta.outputs.version }} | |
| SUPABASE_URL=${{ steps.hosts.outputs.api }} | |
| labels: | | |
| org.opencontainers.image.source=https://github.com/${{ github.repository }} | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.version=${{ needs.meta.outputs.version }} | |
| cache-from: type=gha,scope=web | |
| cache-to: type=gha,scope=web,mode=max | |
| - name: Tag additional refs | |
| run: | | |
| set -euo pipefail | |
| for t in $(echo '${{ needs.meta.outputs.tags }}' | tr ',' ' '); do | |
| [ "$t" = "${{ needs.meta.outputs.version }}" ] && continue | |
| docker buildx imagetools create \ | |
| --tag ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/web:${t} \ | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/web:${{ needs.meta.outputs.version }} | |
| done | |
| build-edge-functions: | |
| needs: meta | |
| runs-on: pawtograder-ci | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push edge-functions image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: charts/pawtograder/images/edge-functions/Dockerfile | |
| push: true | |
| tags: ${{ format('{0}/{1}/edge-functions:{2}', env.REGISTRY, env.REPO_OWNER, needs.meta.outputs.version) }} | |
| build-args: | | |
| GIT_SHA=${{ github.sha }} | |
| VERSION=${{ needs.meta.outputs.version }} | |
| cache-from: type=gha,scope=edge-functions | |
| cache-to: type=gha,scope=edge-functions,mode=max | |
| - name: Tag additional refs | |
| run: | | |
| set -euo pipefail | |
| for t in $(echo '${{ needs.meta.outputs.tags }}' | tr ',' ' '); do | |
| [ "$t" = "${{ needs.meta.outputs.version }}" ] && continue | |
| docker buildx imagetools create \ | |
| --tag ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/edge-functions:${t} \ | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/edge-functions:${{ needs.meta.outputs.version }} | |
| done | |
| build-migrations: | |
| needs: meta | |
| runs-on: pawtograder-ci | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push migrations image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: charts/pawtograder/images/migrations/Dockerfile | |
| push: true | |
| tags: ${{ format('{0}/{1}/migrations:{2}', env.REGISTRY, env.REPO_OWNER, needs.meta.outputs.version) }} | |
| build-args: | | |
| GIT_SHA=${{ github.sha }} | |
| VERSION=${{ needs.meta.outputs.version }} | |
| cache-from: type=gha,scope=migrations | |
| cache-to: type=gha,scope=migrations,mode=max | |
| - name: Tag additional refs | |
| run: | | |
| set -euo pipefail | |
| for t in $(echo '${{ needs.meta.outputs.tags }}' | tr ',' ' '); do | |
| [ "$t" = "${{ needs.meta.outputs.version }}" ] && continue | |
| docker buildx imagetools create \ | |
| --tag ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/migrations:${t} \ | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/migrations:${{ needs.meta.outputs.version }} | |
| done | |
| build-maintenance: | |
| needs: meta | |
| runs-on: pawtograder-ci | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push maintenance image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: charts/pawtograder/images/maintenance/Dockerfile | |
| push: true | |
| tags: ${{ format('{0}/{1}/maintenance:{2}', env.REGISTRY, env.REPO_OWNER, needs.meta.outputs.version) }} | |
| build-args: | | |
| GIT_SHA=${{ github.sha }} | |
| VERSION=${{ needs.meta.outputs.version }} | |
| cache-from: type=gha,scope=maintenance | |
| cache-to: type=gha,scope=maintenance,mode=max | |
| - name: Tag additional refs | |
| run: | | |
| set -euo pipefail | |
| for t in $(echo '${{ needs.meta.outputs.tags }}' | tr ',' ' '); do | |
| [ "$t" = "${{ needs.meta.outputs.version }}" ] && continue | |
| docker buildx imagetools create \ | |
| --tag ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/maintenance:${t} \ | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/maintenance:${{ needs.meta.outputs.version }} | |
| done | |
| # supabase/postgres + a SHA-pinned mc, so the nightly dump does not fetch its | |
| # uploader from a third party on the run that has to work — the dependency | |
| # that broke backups on 2026-09-12, when MinIO archived the open-source mc | |
| # project and dl.min.io began returning 410 for every binary. | |
| # | |
| # Its own job, not a step inside build-maintenance, so that it (a) gets the | |
| # same "Tag additional refs" aliasing every other image gets — without it | |
| # `backup:<channel>-latest` never exists and the CronJobs ImagePullBackOff on | |
| # the next run — and (b) can gate publish-chart on its own. | |
| build-backup: | |
| needs: meta | |
| runs-on: pawtograder-ci | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build & push backup image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: charts/pawtograder/images/backup/Dockerfile | |
| push: true | |
| tags: ${{ format('{0}/{1}/backup:{2}', env.REGISTRY, env.REPO_OWNER, needs.meta.outputs.version) }} | |
| build-args: | | |
| GIT_SHA=${{ github.sha }} | |
| VERSION=${{ needs.meta.outputs.version }} | |
| cache-from: type=gha,scope=backup | |
| cache-to: type=gha,scope=backup,mode=max | |
| - name: Tag additional refs | |
| run: | | |
| set -euo pipefail | |
| for t in $(echo '${{ needs.meta.outputs.tags }}' | tr ',' ' '); do | |
| [ "$t" = "${{ needs.meta.outputs.version }}" ] && continue | |
| docker buildx imagetools create \ | |
| --tag ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/backup:${t} \ | |
| ${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/backup:${{ needs.meta.outputs.version }} | |
| done | |
| publish-chart: | |
| # Note: build-web is intentionally not a dep — on v* tag pushes the web | |
| # image is not built here (URLs would have to be baked at tag time and | |
| # we don't know the consumer's hostname). The chart references images | |
| # by tag only, so it's fine to publish the chart without a same-tag | |
| # web image; consumers of the chart override web.image.tag to a | |
| # workflow_dispatch-built image keyed to their environment. | |
| needs: [meta, build-edge-functions, build-migrations, build-backup] | |
| if: needs.meta.outputs.is_tag == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: azure/setup-helm@v4 | |
| with: | |
| version: v3.14.4 | |
| - name: Lint chart | |
| run: helm lint charts/pawtograder | |
| - name: Package chart | |
| run: helm package charts/pawtograder --version ${{ needs.meta.outputs.chart_version }} --app-version ${{ needs.meta.outputs.version }} | |
| - name: Login to ghcr.io | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ${{ env.REGISTRY }} --username ${{ github.actor }} --password-stdin | |
| - name: Push chart to OCI | |
| run: | | |
| set -euo pipefail | |
| helm push pawtograder-${{ needs.meta.outputs.chart_version }}.tgz \ | |
| oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/charts | |
| # Auto-roll the long-lived staging environment forward to the images this | |
| # push just built. This is the piece that was missing: release-images used to | |
| # only build+push, leaving staging on hand-tagged `staging-manual-*` images | |
| # that someone had to `helm upgrade` by hand. Now every push to `staging` | |
| # deploys itself. | |
| # | |
| # NON-DESTRUCTIVE on purpose: a plain `helm upgrade` (no uninstall, no PVC | |
| # delete). The wipe-and-reinstall flow lives in scripts/redeploy-staging.sh | |
| # and is never run from CI. The per-revision migrations Job runs as part of | |
| # the upgrade and `--wait` blocks until it — and every Deployment — is | |
| # healthy, so a failed migration or rollout turns this job red. | |
| deploy-staging: | |
| env: | |
| # Raise the client-side API throttle for every helm call in this job | |
| # (upgrade, the recovery uninstall/rollback, smoke test, teardown). | |
| # | |
| # Helm defaults to burst 100 and QPS 0 — and QPS 0 means client-go's | |
| # default of 5 req/s. The initial apply spends the burst, then everything | |
| # after it (notably --wait/--wait-for-jobs polling every resource in the | |
| # release) crawls at 5 req/s until the operation context expires: | |
| # UPGRADE FAILED: client rate limiter Wait returned an error | |
| # That error is the client queueing behind its own limiter, not the API | |
| # server rejecting anything, so the fix is on this side. | |
| HELM_QPS: "50" | |
| HELM_BURST_LIMIT: "300" | |
| needs: [meta, build-web, build-edge-functions, build-migrations, build-maintenance, build-backup] | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/staging' | |
| # MUST run on the self-hosted, in-network runner. The cluster API | |
| # (talos.ripley.cloud:6443) is a private 10.201.2.x address, unreachable | |
| # from GitHub-hosted runners — `helm upgrade` there fails instantly with | |
| # "Kubernetes cluster unreachable: ... i/o timeout". build-web reaches the | |
| # same cluster from this runner to read the anon-key secret, so it's the | |
| # proven path for the helm upgrade too. | |
| runs-on: pawtograder-ci | |
| # Serialize deploys so two staging pushes landing close together can't run | |
| # overlapping `helm upgrade`s against the same release. | |
| concurrency: | |
| group: deploy-staging | |
| cancel-in-progress: false | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| # This job handles cluster credentials; no later step needs the git | |
| # token, so don't leave it persisted in .git/config. | |
| persist-credentials: false | |
| - uses: azure/setup-helm@v4 | |
| with: | |
| version: v3.14.4 | |
| - name: Write kubeconfig | |
| env: | |
| KUBECONFIG_B64: ${{ secrets.KUBECONFIG_BASE64 }} | |
| run: | | |
| set -euo pipefail | |
| [ -n "$KUBECONFIG_B64" ] || { echo "::error::KUBECONFIG_BASE64 secret is required to deploy staging"; exit 1; } | |
| mkdir -p "$HOME/.kube" | |
| echo "$KUBECONFIG_B64" | base64 -d > "$HOME/.kube/config" | |
| chmod 600 "$HOME/.kube/config" | |
| - name: Recover stuck helm release | |
| # If a prior deploy was interrupted mid-`helm upgrade` (runner crash/OOM | |
| # — cancel-in-progress is off here, so a cancelled run can't be the | |
| # cause), the release is left in a `pending-*` state and every later | |
| # upgrade aborts with "another operation (install/upgrade/rollback) is | |
| # in progress". Clear it. Unlike previews we do NOT uninstall — staging | |
| # is long-lived and uninstall means downtime — so roll back to the last | |
| # good revision, which clears the lock and keeps staging serving. | |
| run: | | |
| set -euo pipefail | |
| status=$(helm status pawtograder -n pawtograder-staging 2>/dev/null | sed -n 's/^STATUS: //p' || true) | |
| echo "current helm release status: ${status:-<none>}" | |
| case "$status" in | |
| pending-install) | |
| echo "::warning::staging release stuck in pending-install — uninstalling the aborted install (PVCs survive)" | |
| helm uninstall pawtograder -n pawtograder-staging --wait --timeout 10m || true | |
| ;; | |
| pending-upgrade|pending-rollback) | |
| last=$(helm history pawtograder -n pawtograder-staging -o json 2>/dev/null \ | |
| | jq -r '[.[] | select(.status=="deployed")] | last | .revision // empty' 2>/dev/null || true) | |
| if [ -n "${last:-}" ]; then | |
| echo "::warning::staging release stuck in '$status' — rolling back to last deployed revision $last to clear the lock" | |
| helm rollback pawtograder "$last" -n pawtograder-staging --wait --timeout 10m || true | |
| else | |
| echo "::warning::staging release stuck in '$status' — rolling back to previous revision" | |
| helm rollback pawtograder -n pawtograder-staging --wait --timeout 10m || true | |
| fi | |
| ;; | |
| esac | |
| - name: helm upgrade pawtograder (staging) | |
| # helm/kubectl pick up $HOME/.kube/config (written above) by default. | |
| env: | |
| VERSION: ${{ needs.meta.outputs.version }} | |
| run: | | |
| set -euo pipefail | |
| # Pin all three app images to the exact tag built from this SHA | |
| # (staging-<sha>) for traceability + easy rollback, overriding the | |
| # defaults in values-staging.yaml. | |
| helm upgrade --install pawtograder charts/pawtograder \ | |
| --namespace pawtograder-staging \ | |
| -f charts/pawtograder/examples/values-staging.yaml \ | |
| --set web.image.tag="$VERSION" \ | |
| --set edgeFunctions.image.tag="$VERSION" \ | |
| --set migrations.image.tag="$VERSION" \ | |
| --set backup.image.tag="$VERSION" \ | |
| --set maintenance.enabled=true \ | |
| --set maintenance.image.repository=ghcr.io/pawtograder/maintenance \ | |
| --set maintenance.image.tag="$VERSION" \ | |
| --wait \ | |
| --wait-for-jobs \ | |
| --timeout 15m | |
| - name: helm smoke test pawtograder (staging) | |
| run: helm test pawtograder -n pawtograder-staging --timeout 10m | |
| - name: Post-deploy status | |
| if: always() | |
| run: helm status pawtograder -n pawtograder-staging --kubeconfig "$HOME/.kube/config" || true |