perf(async-worker): keep n messages in flight instead of draining a batch at a time

charts/pawtograder/templates/_edge-functions-workload.tpl

@@ -303,6 +303,12 @@ spec:
               value: {{ $ctx.Values.edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg | quote }}
             - name: GITHUB_ASYNC_WORKER_ORG_SLOT_LEASE_TTL_SECONDS
               value: {{ $ctx.Values.edgeFunctions.githubAsyncWorker.orgSlotLeaseTtlSeconds | quote }}
+            # Continuous-refill KILL SWITCH, 1 = on (shipped). Only meaningful on
+            # the org-leased path. It exists so the drain SHAPE can be rolled
+            # back without a code deploy and without setting orgSlotGlobalCap: 0,
+            # which would also give up per-org leaseholders.
+            - name: GITHUB_ASYNC_WORKER_ORG_SLOT_CONTINUOUS_REFILL
+              value: {{ $ctx.Values.edgeFunctions.githubAsyncWorker.orgSlotContinuousRefill | quote }}
             # Latency histogram bounds in seconds. The top finite bucket must be
             # >= worker.timeoutMs/1000 or every request that hits the worker
             # timeout lands in +Inf and the upper quantiles become an

charts/pawtograder/templates/validations.yaml

@@ -273,26 +273,34 @@ reason resolveAsyncWorkerTuning only records these issues when the feature is on
 {{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotGlobalCap must be an integer 0-8 (got %q). 0 disables per-org leaseholders; a non-numeric value would be clamped back to 0 at runtime, so the deploy would silently keep the single-leaseholder path it was meant to replace." $capRaw) -}}
 {{- end -}}
 {{- if not (regexMatch "^[0-9]+$" $mpoRaw) -}}
-{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg must be an integer 1-2 (got %q)." $mpoRaw) -}}
+{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg must be an integer 1-4 (got %q)." $mpoRaw) -}}
 {{- end -}}
 {{- if not (regexMatch "^[0-9]+$" $ttlRaw) -}}
 {{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotLeaseTtlSeconds must be an integer 45-300 (got %q)." $ttlRaw) -}}
 {{- end -}}
+{{- $refillRaw := $gaw.orgSlotContinuousRefill | toString -}}
+{{- if not (regexMatch "^[0-9]+$" $refillRaw) -}}
+{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotContinuousRefill must be the integer 0 or 1 (got %q). It is deliberately an INTEGER and not a true/false string: Boolean(\"false\") is true in JS, so a string-valued kill switch turns a typo into silent truthiness. This rule refuses the render so a chart user never reaches that; on the Helm-bypass paths this rule cannot see, resolveAsyncWorkerTuning FAILS SAFE to 0 (off) for a value that is present but unusable, while an ABSENT value still resolves to 1 (on)." $refillRaw) -}}
+{{- end -}}
+{{- $refill := int $refillRaw -}}
+{{- if or (lt $refill 0) (gt $refill 1) -}}
+{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotContinuousRefill=%d is outside 0-1. 1 (the default) keeps drainConcurrency messages in flight and claims the shortfall as each settles; 0 restores the batch-and-re-claim shape, which measured 5.20 effective concurrency of a possible 8 on the 2026-09-13 burst because every batch waits on its straggler. This is a KILL SWITCH, so the range is binary on purpose — there is no low-water-mark parameter in drainWithContinuousRefill for an intermediate value to mean, and a number a values file renders but nothing reads is worse than no knob." $refill) -}}
+{{- end -}}
 {{- $cap := int $capRaw -}}
 {{- $mpo := int $mpoRaw -}}
 {{- $ttl := int $ttlRaw -}}
 {{- if gt $cap 8 -}}
 {{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=%d is above 8. Each leaseholder is a RESIDENT isolate holding one admission slot and one worker.memoryLimitMb heap, and the pod memory budget is written as (maxParallelism x worker.memoryLimitMb); the cap is fleet-wide while maxParallelism (%d) is per-pod, so 8 is the worst-case-colocation number. Note also the slot POOL is seeded by migration (async_calls 64, async_calls_low_priority 16) — raising the cap past a pool would silently cap the budget at the pool size instead." $cap (int .Values.edgeFunctions.maxParallelism)) -}}
 {{- end -}}
-{{- if or (lt $mpo 1) (gt $mpo 2) -}}
-{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=%d is outside 1-2. In-flight per org is orgSlotMaxPerOrg x drainConcurrency against ONE org's content limiter (reservoir 40 / maxConcurrent 40 / refresh 40 per 60s). The RESERVOIR binds before the concurrency pool does: at the post-permission-sync-fix create_repo p50 of 23.1s (2026-09-13, 242 messages) 8 in flight is ~21 starts/min of a 40/min refresh, and 16 would be ~42/min — over budget. 8 is also the already-argued 20%% of the 40-slot pool, which is what leaves org invitations and handout syncs their headroom." $mpo) -}}
+{{- if or (lt $mpo 1) (gt $mpo 4) -}}
+{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=%d is outside 1-4. In-flight per org is orgSlotMaxPerOrg x drainConcurrency against ONE org's content limiter (reservoir 40 / maxConcurrent 40 / refresh 40 per 60s). The RESERVOIR binds before the concurrency pool does: with c creations in flight at d seconds each the start rate is 60c/d per minute, and measured create_repo p50 over the 3 days to 2026-09-14 on Khoury prod runs 23.1s (neu-cs2000) to 46.0s (Khoury-CS3650) — so 16 in flight is 41.6/min for the fast org, OVER the 40/min refresh, and 20.9/min for the slow one. The ceiling is 4 so the slow-org case is expressible; it is NOT a recommendation, because this knob has no per-org dimension and one setting applies to both. Recommended is 2 (8 in flight), which is also the already-argued 20%% of the 40-slot pool that leaves org invitations and handout syncs their headroom." $mpo) -}}
 {{- end -}}
 {{- if or (lt $ttl 45) (gt $ttl 300) -}}
-{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotLeaseTtlSeconds=%d is outside 45-300. Below 45s the renewal timer — which shares an event loop with the handlers — can reap a slot from a leaseholder that is still working, because the longest single message measured was 32.2s (2026-09-13). Above 300s the TTL becomes the slot-recovery latency for a leaseholder that really died, and it must not outlive the messages it is holding." $ttl) -}}
+{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotLeaseTtlSeconds=%d is outside 45-300. Below 45s an EVENT-LOOP STALL can let the lease lapse while the leaseholder is still working, after which a second leaseholder enters the same org and the per-org cap is breached. Note this floor is NOT about message duration (it used to be argued that way, from a 32.2s figure that was one org on one day): renewal is an independent setInterval at TTL/3 and the handlers await on GitHub I/O, so the real longest message — 97.7s create_repo over 489 samples from 2026-09-11, and 1.5%% of all messages exceed the shipped 60s TTL — renews through fine. 45s tolerates a 45s unyielding stall and leaves a 15s renewal interval against a millisecond Postgres round trip. Above 300s the TTL becomes the slot-recovery latency for a leaseholder that really died, and it must not outlive the messages it is holding." $ttl) -}}
 {{- end -}}
 {{- if gt $cap 0 -}}
-{{- if gt (mul $mpo $n) 8 -}}
-{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=%d x drainConcurrency=%d is %d in flight for a SINGLE org, above the 8 ceiling. That ceiling is 20%% of the fleet-wide per-org content limiter (40 concurrent, shared with org invitations and handout syncs), and the 40-per-minute reservoir is the tighter of the two. Lower orgSlotMaxPerOrg rather than drainConcurrency — drainConcurrency is already coherent with visibilityTimeoutSeconds and worker.timeoutMs, and lowering it would break that pairing to fix a different budget." $mpo $n (mul $mpo $n)) -}}
+{{- if gt (mul $mpo $n) 16 -}}
+{{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=%d x drainConcurrency=%d is %d in flight for a SINGLE org, above the 16 ceiling. That ceiling is about the fleet-wide per-org content limiter (40 concurrent AND 40 starts per 60s, shared with org invitations and handout syncs), and the RESERVOIR is the tighter of the two: at the fastest measured create_repo p50 (23.1s, neu-cs2000, 2026-09-14) even 16 in flight is 41.6 starts/min against a 40/min refresh, so 16 is already the edge and not headroom. Lower orgSlotMaxPerOrg rather than drainConcurrency — drainConcurrency is already coherent with visibilityTimeoutSeconds and worker.timeoutMs, and lowering it would break that pairing to fix a different budget. This ceiling is deliberately NOT edgeFunctions.githubAsyncWorker.drainConcurrency's own maximum of 8: that one bounds one isolate's batch (heap, and the n x 120 visibility-timeout model), this one bounds one org's share of a GitHub rate limit." $mpo $n (mul $mpo $n)) -}}
 {{- end -}}
 {{- if gt $mpo $cap -}}
 {{- fail (printf "edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=%d exceeds orgSlotGlobalCap=%d, so the per-org ceiling can never be reached and the value is a lie about what this deploy does. Set orgSlotGlobalCap >= orgSlotMaxPerOrg." $mpo $cap) -}}

charts/pawtograder/tests/render-guardrails.sh

@@ -1540,13 +1540,17 @@ assert_env_value "recommended prod cap renders" \
 
 # maxPerOrg x drainConcurrency is in-flight work against ONE org's content
 # limiter (40 concurrent / 40 per minute, shared with org invitations and
-# handout syncs). Past 8 the 40/min reservoir is the binding constraint, and
-# the symptom is students' org invitations convoying behind repo creations —
+# handout syncs). The 40/min RESERVOIR is the binding constraint, and the
+# symptom is students' org invitations convoying behind repo creations —
 # nothing about it looks like a queue problem.
-assert_refused "refuses more than 8 in flight for one org" \
+#
+# The ceiling was 8 until 2026-09-14 and is now 16, so this case is 4 x 8 = 32.
+# The product ceiling is deliberately not drainConcurrency's own maximum of 8:
+# that bounds one isolate's batch, this bounds one org's share of a rate limit.
+assert_refused "refuses more than 16 in flight for one org" \
   "in flight for a SINGLE org" \
   --set edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=8 \
-  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=2 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=4 \
   --set edgeFunctions.githubAsyncWorker.drainConcurrency=8 \
   --set edgeFunctions.githubAsyncWorker.visibilityTimeoutSeconds=960 \
   --set edgeFunctions.worker.timeoutMs=960000 \
@@ -2032,6 +2036,124 @@ assert_renders "permits a msg_id warning inside the band" \
   --set monitoring.enabled=true --set monitoring.prometheusRules.labels.release=kps \
   --set monitoring.prometheusRules.queueMsgIdWarning=2000000000000
 
+echo
+
+echo "== the per-org in-flight ceiling is a RANGE (raised 8 -> 16, 2026-09-14) =="
+# orgSlotMaxPerOrg was pinned at 2 (8 in flight at the shipped drainConcurrency
+# of 4), which was simultaneously the measured recommendation AND the hard
+# ceiling — so a deployment whose create_repo is slow had no way to ask for
+# more. The ceiling is now 4 / 16 in flight. What must stay true:
+#
+#   * the new maximum actually renders, or the range is a lie;
+#   * one past it is still refused, or there is no ceiling at all;
+#   * the maxPerOrg x drainConcurrency product is still enforced, at 16 rather
+#     than 8 — the two constraints are independent and only the product moved;
+#   * and the SHIPPED DEFAULTS still pass their own validation, which is the
+#     property that keeps "higher values are expressible" from quietly becoming
+#     "higher values are what you get".
+
+# The new maximum renders AND reaches the container. A range check that passes
+# while the value is dropped on the floor guards nothing.
+assert_env_value "the new maxPerOrg maximum renders and reaches the pod" \
+  templates/edge-functions.yaml GITHUB_ASYNC_WORKER_ORG_SLOT_MAX_PER_ORG 4 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=8 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=4
+
+# One past the ceiling. 5 x 4 = 20 in flight would also trip the product rule,
+# but the RANGE rule runs first and is the one being pinned here.
+assert_refused "refuses maxPerOrg one past the new ceiling" \
+  "is outside 1-4" \
+  --set edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=8 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=5
+
+# The product ceiling moved WITH the range, not instead of it: 16 in flight is
+# now legal (this is the case that used to be refused), and 24 is not. Both
+# halves matter — asserting only the refusal would pass even if the ceiling had
+# never moved.
+assert_renders "permits exactly 16 in flight for one org" \
+  --set edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=8 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=2 \
+  --set edgeFunctions.githubAsyncWorker.drainConcurrency=8 \
+  --set edgeFunctions.githubAsyncWorker.visibilityTimeoutSeconds=960 \
+  --set edgeFunctions.worker.timeoutMs=960000 \
+  --set edgeFunctions.gracefulExitTimeoutSeconds=1000 \
+  --set edgeFunctions.terminationGracePeriodSeconds=1030
+
+assert_refused "enforces the product ceiling at the new value, not the new range" \
+  "above the 16 ceiling" \
+  --set edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=8 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=3 \
+  --set edgeFunctions.githubAsyncWorker.drainConcurrency=8 \
+  --set edgeFunctions.githubAsyncWorker.visibilityTimeoutSeconds=960 \
+  --set edgeFunctions.worker.timeoutMs=960000 \
+  --set edgeFunctions.gracefulExitTimeoutSeconds=1000 \
+  --set edgeFunctions.terminationGracePeriodSeconds=1030
+
+# THE SHIPPED DEFAULTS STILL PASS THEIR OWN VALIDATION. Raising a bound is the
+# classic way to ship a values.yaml that the chart's own rules reject on the
+# next unrelated render, and nothing else here renders the chart with no --set
+# at all.
+assert_renders "the shipped defaults pass the raised validation unchanged" \
+  --set global.environment=production
+assert_env_value "the shipped default is still 1, not the new ceiling" \
+  templates/edge-functions.yaml GITHUB_ASYNC_WORKER_ORG_SLOT_MAX_PER_ORG 1
+
+# The documented recommendation is also still only a recommendation: 2 is what
+# values.yaml tells an operator to use and it must keep rendering, distinct
+# from the 4 the range now permits.
+assert_env_value "the recommended value (2) is unchanged by the raised ceiling" \
+  templates/edge-functions.yaml GITHUB_ASYNC_WORKER_ORG_SLOT_MAX_PER_ORG 2 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=8 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=2
+
+echo
+
+echo
+
+echo "== continuous refill must ship ON and stay switchable without a code deploy =="
+# Continuous refill changes the org-leased drain SHAPE: n messages kept in
+# flight with the shortfall claimed as each settles, instead of a batch drained
+# and re-claimed. It ships on, because the batch shape measured 5.20 effective
+# concurrency of a possible 8.
+#
+# The property this guards is the ROLLBACK PATH. Before this knob, undoing the
+# drain shape meant orgSlotGlobalCap: 0, which also switches off per-org
+# leaseholders and gives back the cross-org throughput that is already working
+# in production. A kill switch that cannot be reached, or that is not actually
+# wired to the pod, is not a rollback path at all.
+
+assert_env_value "continuous refill ships ON" \
+  templates/edge-functions.yaml GITHUB_ASYNC_WORKER_ORG_SLOT_CONTINUOUS_REFILL 1
+
+# The OFF position must render AND reach the container. This is the assertion
+# that makes it a kill switch rather than a comment: a range check that passes
+# while the value is dropped on the floor rolls nothing back.
+assert_env_value "continuous refill can be switched OFF and reaches the pod" \
+  templates/edge-functions.yaml GITHUB_ASYNC_WORKER_ORG_SLOT_CONTINUOUS_REFILL 0 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotContinuousRefill=0
+
+# Turning refill off must NOT require giving up per-org leaseholders — that is
+# the entire reason the knob exists, so pin the combination explicitly.
+assert_env_value "refill OFF keeps per-org leaseholders ON" \
+  templates/edge-functions.yaml GITHUB_ASYNC_WORKER_ORG_SLOT_GLOBAL_CAP 8 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotGlobalCap=8 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotMaxPerOrg=2 \
+  --set edgeFunctions.githubAsyncWorker.orgSlotContinuousRefill=0
+
+# A string boolean is the failure mode this knob is typed against: Boolean("false")
+# is true in JS, so "false" would fail OPEN and the switch would not switch.
+assert_refused "refuses a string boolean, which would fail OPEN at runtime" \
+  "must be the integer 0 or 1" \
+  --set-string edgeFunctions.githubAsyncWorker.orgSlotContinuousRefill=false
+
+# Out of range: 2 is not "extra on". There is no low-water-mark parameter for an
+# intermediate value to mean, so the range is binary on purpose.
+assert_refused "refuses a value outside the binary range" \
+  "is outside 0-1" \
+  --set edgeFunctions.githubAsyncWorker.orgSlotContinuousRefill=2
+
+echo
+
 if [ "$FAILED" -ne 0 ]; then
   echo "GUARD-RAIL TESTS FAILED"
   exit 1