E2E nightly #122
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Nightly L4 Kind matrix (deduped — L3 export/object-store runs in ci.yaml test-integration). | |
| name: E2E nightly | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| run_scale_jobs: | |
| description: Run 8-core scale jobs (requires ubuntu-latest-8-cores — currently unavailable) | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: e2e-nightly-${{ github.workflow }}-${{ github.ref }} | |
| # HY-06: never cancel in-flight runs on main (would leave CI-UNKNOWN); PR runs still supersede. | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| # Contract lock for COV-90-S14 wiring (no Kind). | |
| finalizer-cleanup-meta: | |
| name: finalizer-cleanup-meta | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Verify finalizer-cleanup assert + workflow wiring | |
| run: bash hack/test/finalizer_cleanup_e2e_test.sh | |
| # Contract lock for multitenant fail-dump + collecting-wait harden (no Kind). | |
| multitenant-meta: | |
| name: multitenant-meta | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Verify multitenant assert diagnostics + collecting wait | |
| run: bash hack/test/e2e_mt_repro_harden_test.sh | |
| # Each scenario gets its own Kind cluster (setup → scenario → teardown) so failures | |
| # are isolated and parallelized. fail-fast: false keeps other scenarios running. | |
| e2e-scenario: | |
| name: e2e-${{ matrix.scenario }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - scenario: smoke | |
| script: hack/kind/e2e/smoke.sh | |
| run_mode: bash | |
| - scenario: git-export | |
| script: hack/e2e/git-export-assert.sh | |
| run_mode: repo_root | |
| - scenario: tenant-mode | |
| script: hack/e2e/tenant-mode.sh | |
| run_mode: repo_root | |
| - scenario: multitenant | |
| script: hack/e2e/multitenant.sh | |
| run_mode: repo_root | |
| - scenario: finalizer-cleanup | |
| script: hack/e2e/finalizer-cleanup-assert.sh | |
| run_mode: repo_root | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: ./.github/actions/kind-e2e-setup | |
| with: | |
| scenario-script: ${{ matrix.script }} | |
| run-mode: ${{ matrix.run_mode }} | |
| wait-timeout: 300s | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| git-export-test-repo: ${{ vars.GIT_EXPORT_TEST_REPO }} | |
| - name: Summarize nightly failure | |
| if: failure() | |
| shell: bash | |
| run: | | |
| { | |
| echo "::error title=Nightly e2e failed::Scenario '${{ matrix.scenario }}' failed." | |
| echo "Triage: check inventory reconciliation timeouts, kind.sigs.k8s.io download flakes," | |
| echo "and Helm install waits. Re-run via workflow_dispatch or \`task test:e2e\` locally." | |
| echo "Triage guide: docs/development/testing.md" | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| e2e-bench: | |
| name: e2e-bench | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: ./.github/actions/go-cache | |
| with: | |
| envtest: true | |
| - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 | |
| with: | |
| version: 3.51.1 | |
| - name: Micro-benchmarks | |
| run: task bench | |
| - name: Upload bench artifact | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: bench-latest | |
| path: artifacts/bench/latest.txt | |
| retention-days: 14 | |
| if-no-files-found: warn | |
| # Advisory race detector for unit + envtest (TEST-02 / HY-07). | |
| # continue-on-error: does not block nightly green; file issues from the summary excerpt. | |
| # COVERAGE_MIN=0: racing is the signal — do not fail the advisory job on the coverage floor. | |
| race: | |
| name: race (advisory) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| continue-on-error: true | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: ./.github/actions/go-cache | |
| with: | |
| envtest: true | |
| - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 | |
| with: | |
| version: 3.51.1 | |
| - name: Unit + envtest under -race | |
| id: race | |
| env: | |
| CI: "true" | |
| CGO_ENABLED: "1" | |
| COVERAGE_MIN: "0" | |
| run: | | |
| set +e | |
| task coverage:race 2>&1 | tee race.log | |
| status=${PIPESTATUS[0]} | |
| if grep -q 'WARNING: DATA RACE' race.log; then | |
| { | |
| echo "## Race detector findings" | |
| echo "" | |
| echo "DATA RACE detected. Excerpt:" | |
| echo '```' | |
| grep -A 40 'WARNING: DATA RACE' race.log | head -80 | |
| echo '```' | |
| echo "" | |
| echo "File an issue labeled \`race\`/\`flake\` referencing this run (do not ignore)." | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| exit 1 | |
| fi | |
| if [ "$status" -ne 0 ]; then | |
| { | |
| echo "## Race job failed (non-race)" | |
| echo "" | |
| echo "Exit status ${status} without a DATA RACE marker — likely timeout or compile failure." | |
| echo '```' | |
| tail -n 80 race.log | |
| echo '```' | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| exit "$status" | |
| fi | |
| echo "Race detector clean." >> "${GITHUB_STEP_SUMMARY}" | |
| - name: Upload race log | |
| if: failure() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: race-log | |
| path: race.log | |
| retention-days: 14 | |
| if-no-files-found: warn | |
| # PERF-FIX-04: this used to be `load-test-10k` on ubuntu-latest-8-cores, which read | |
| # as cluster-scale evidence. It never was: it is a single-threaded, in-process | |
| # extractor micro-benchmark with no API server, cluster, sinks or export path. | |
| # Renamed, moved to a plain runner, and made to actually run every nightly. | |
| # In-cluster scale evidence is the `scale-envtest-10k` job below. | |
| extract-budget: | |
| name: extract-budget | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| # No envtest cache on purpose: this job never starts an API server. | |
| - uses: ./.github/actions/go-cache | |
| - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 | |
| with: | |
| version: 3.51.1 | |
| - name: Extractor hot-path budget (micro-benchmark, not a scale tier) | |
| run: task extract-budget | |
| env: | |
| CI: "true" | |
| scale-envtest-10k: | |
| name: scale-envtest-10k | |
| # Skipped by default: ubuntu-latest-8-cores runners unavailable (jobs never leave queue). | |
| # Re-enable via workflow_dispatch with run_scale_jobs=true once org runners are provisioned. | |
| if: github.event_name == 'workflow_dispatch' && inputs.run_scale_jobs == true | |
| runs-on: ubuntu-latest-8-cores | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: ./.github/actions/go-cache | |
| with: | |
| envtest: true | |
| - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 | |
| with: | |
| version: 3.51.1 | |
| - name: Extended envtest scale (10k) | |
| run: go test -run TestEngine_ScaleEnvtestOptIn -count=1 -timeout=35m ./internal/collect/... | |
| env: | |
| KOLECT_SCALE_TEST_MAX: "10000" | |
| CI: "true" | |