perf(dal): upsert scan telemetry in one statement per table (#245) #809

Workflow file for this run

	name: CI

	# One run per change, and a button for everything else.
	#
	# `push: [""]` alongside `pull_request` ran the whole suite twice** for every
	# push to a branch with an open PR — two runners, same commit, simultaneously.
	# That was not only double spend: it took two independent samples of every
	# timing-sensitive gate, so the slow-test budget had two chances to trip per
	# push. It did, repeatedly, on commits whose sibling run was green — a red X on a
	# PR for reasons that had nothing to do with the change under review.
	#
	# So `push` is now `main` only, which is the post-merge run, and `pull_request`
	# is the pre-merge one. Exactly one of them fires for any given change.
	#
	# `pull_request` is what the branch ruleset's required checks report from, so it
	# cannot be dropped: without it no check would ever report on a PR and every PR
	# would block forever.
	#
	# `workflow_dispatch` covers the case the `push: ["**"]` trigger was really
	# serving — running the full suite on a branch before opening a PR. It is the
	# same job set, deliberately: a run that skips tiers cannot tell you the branch
	# is mergeable, which is the reason to press the button.
	#
	# `develop` is gone from `pull_request` because no such branch exists; it was
	# dead config that read as though a second integration branch were gated.
	#
	# `pull_request` carries no branch filter, and that is load-bearing rather
	# than an omission. The filter names the branch a PR is merging into, so
	# `branches: [main]` silently excluded every stacked PR — a PR opened against
	# another feature branch matched nothing and ran no CI at all. Retargeting it to
	# `main` later does not help: that fires `edited`, which is not in this trigger's
	# default types (`opened`, `synchronize`, `reopened`).
	#
	# The failure mode is worse than an absent run, because `gh pr checks` reads as
	# green: the required jobs are simply missing, while `Codacy` and `PR title`
	# report and pass. Only the ruleset refusing to merge reveals it. Stacked PRs are
	# how work lands here, so the branch filter cost coverage exactly where review is
	# already hardest and bought nothing — the double-run it looked like it was
	# preventing came from `push: ["**"]`, which is fixed above and stays fixed.
	on:
	push:
	branches: [main]
	pull_request:
	workflow_dispatch:

	env:
	CARGO_TERM_COLOR: always
	RUST_BACKTRACE: 1

	jobs:
	fmt:
	name: Format check
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	- uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
	with:
	toolchain: stable
	components: rustfmt
	- name: cargo fmt --check
	working-directory: dpp-engine
	run: cargo fmt --all --check

	clippy:
	name: Clippy
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	- uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
	with:
	toolchain: stable
	components: clippy
	- uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2
	with:
	workspaces: dpp-engine
	# Linted with the `integration-tests` features ON, which folds in what a
	# separate compile-only job used to do.
	#
	# Those suites are otherwise compiled nowhere except the Docker tiers,
	# which need `test-unit` to pass first — so a stale import in one surfaced
	# late, or not at all when Docker was unavailable. Clippy with the features
	# on type-checks them and lints them, which a `cargo check` did not.
	#
	# Safe to fold rather than run twice: features are additive and the
	# workspace has zero `cfg(not(feature = "integration-tests"))` code, so
	# nothing compiled by the default build is skipped by this one. If that
	# ever stops being true, the default-feature pass has to come back.
	#
	# Features are named per package rather than via `--all-features`, which
	# would also enable `cli/desktop` (pulls rfd's ashpd/Wayland subtree, kept
	# out of server-side builds on purpose — see cli/Cargo.toml) and
	# `dpp-plugin-host/wasm-fixture-tests` (needs the wasm32-wasip1 target and
	# a nested cargo build). Neither is a gap this closes.
	- name: cargo clippy
	working-directory: dpp-engine
	run: \|
	cargo clippy --workspace --all-targets \
	--features dpp-dal/integration-tests,dpp-vault/integration-tests,dpp-plugin-host/integration-tests,dpp-node/integration-tests \
	-- -D warnings

	test-unit:
	name: Unit tests
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	- uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2
	with:
	workspaces: dpp-engine
	- name: Install cargo-nextest
	uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2
	with:
	tool: nextest
	# Run first, and as its own step, so "the API description no longer
	# matches the code" is a named CI failure rather than one line inside a
	# check called "Unit tests". It is cheap here (the toolchain and cache are
	# already warm) and the `--workspace` run below re-runs it for free.
	#
	# This is only half the enforcement: the test reads
	# `api/openapi.bundled.json`, embedded at compile time, so a stale
	# bundle would let it pass against an outdated spec. The `openapi-lint`
	# job regenerates both bundles and diffs them, which is what closes that.
	# Both jobs must be required checks — either one alone can be satisfied
	# while the API and the code disagree.
	- name: OpenAPI contract — spec vs. the types and routes
	working-directory: dpp-engine
	run: cargo nextest run -p dpp-node --test openapi_contract --no-tests=fail
	# `--workspace` (no integration feature) runs every non-Docker test:
	# lib tests, the `odal` CLI binary tests, and the resolver e2e / JWS
	# verification + security-regression tests. The feature-gated
	# testcontainer tests run in the `test-integration` job below.
	- name: Unit + CLI + resolver tests
	working-directory: dpp-engine
	run: cargo nextest run --workspace --no-tests=fail

	test-integration:
	name: Integration tests (testcontainers)
	runs-on: ubuntu-latest
	needs: [test-unit]
	# One Postgres for the whole job, rather than one container per test.
	#
	# `start_pg` is called from 170 places and used to boot a fresh `postgres:17`
	# each time; measured on this workspace, the tests that did so were 86% of
	# all test time at 12-16s apiece. The harness now clones a per-test database
	# from a migrated template on this server — roughly 190ms — and falls back to
	# starting its own container when `ODAL_TEST_PG_ADMIN_URL` is unset, which is
	# what a bare `cargo nextest run` still does.
	#
	# Docker is still needed: the plugin-host suite uses it, and the migration
	# tests (`start_pg_before`) deliberately keep starting their own server,
	# because a test of a migration needs one the migration has not been applied
	# to.
	services:
	postgres:
	image: postgres:17
	env:
	POSTGRES_USER: postgres
	POSTGRES_PASSWORD: test
	POSTGRES_DB: postgres
	ports:
	- 5432:5432
	options: >-
	--health-cmd pg_isready
	--health-interval 5s
	--health-timeout 5s
	--health-retries 12
	env:
	ODAL_TEST_PG_ADMIN_URL: postgres://postgres:test@127.0.0.1:5432/postgres
	ODAL_TEST_S3_ENDPOINT: http://127.0.0.1:9000
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	- uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2
	with:
	workspaces: dpp-engine
	- name: Install cargo-nextest
	uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2
	with:
	tool: nextest
	- name: Create results dir
	run: mkdir -p dpp-engine/results
	# One MinIO for the whole job, rather than one per S3 test — the same
	# arrangement as the Postgres service above, for the same reason. The four
	# `s3_archive` tests booted their own and spent about ten seconds each
	# doing it, which put them on the wrong side of the slow-test budget
	# whenever the runner was busy. `ODAL_TEST_S3_ENDPOINT` points them here;
	# unset, they still start their own container, so a bare `cargo test`
	# keeps working.
	#
	# A step rather than a `services:` entry because MinIO needs `server /data`
	# as its command, and a service container can set `image`, `env`, `ports`,
	# `volumes` and `options` but not a command. The alternative — an image
	# that runs the server by default — would have CI and a local run testing
	# two different MinIO builds, which is a worse trade than a longer step.
	# The tag is the one `s3_archive.rs` pins for its own container; the two
	# must move together.
	- name: Start MinIO
	run: \|
	docker run -d --name minio \
	-p 9000:9000 \
	-e MINIO_ROOT_USER=minioadmin \
	-e MINIO_ROOT_PASSWORD=minioadmin \
	minio/minio:RELEASE.2025-09-07T16-13-09Z \
	server /data --console-address :9001
	for _ in $(seq 1 30); do
	if curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null 2>&1; then
	echo "minio ready"
	exit 0
	fi
	sleep 1
	done
	echo "minio did not become ready in 30s"
	docker logs minio
	exit 1
	# Docker is pre-installed on ubuntu-latest; testcontainers uses it.
	# The plugin-host suite includes the fuel-exhaustion sandbox test, which
	# aborts on Windows (SEH unwind) but must pass on the Linux deploy target.
	# JUnit output path comes from .config/nextest.toml (profile.default.junit),
	# always target/nextest/default/junit.xml — nextest has no --junit-path flag.
	# `set +e` around the run so a test failure doesn't skip the copy below.
	- name: DAL integration tests
	working-directory: dpp-engine
	run: \|
	set +e
	cargo nextest run -p dpp-dal --features integration-tests --no-tests=fail
	test_exit=$?
	set -e
	cp target/nextest/default/junit.xml results/dal.xml
	exit $test_exit
	- name: Vault integration tests (lifecycle, auth, validation)
	working-directory: dpp-engine
	run: \|
	set +e
	cargo nextest run -p dpp-vault --features integration-tests --no-tests=fail
	test_exit=$?
	set -e
	cp target/nextest/default/junit.xml results/vault.xml
	exit $test_exit
	- name: Plugin host integration tests (incl. fuel sandbox)
	working-directory: dpp-engine
	run: \|
	set +e
	cargo nextest run -p dpp-plugin-host --features integration-tests --no-tests=fail
	test_exit=$?
	set -e
	cp target/nextest/default/junit.xml results/plugin-host.xml
	exit $test_exit
	- name: Node smoke tests
	working-directory: dpp-engine
	run: \|
	set +e
	cargo nextest run -p dpp-node --features integration-tests --no-tests=fail
	test_exit=$?
	set -e
	cp target/nextest/default/junit.xml results/node.xml
	exit $test_exit
	- name: Upload integration test results
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	with:
	name: integration-test-results-${{ github.run_id }}
	path: dpp-engine/results/*.xml
	retention-days: 90
	# Reads the JUnit written above rather than re-running anything, so it
	# measures what happened and cannot itself be flaky. Enforced here and not
	# locally: the slowest test on a Linux runner is ~5s, while the same test
	# behind Docker Desktop reaches 17s, so a dev machine would fail honest
	# work. `slow-timeout` in .config/nextest.toml warns there instead.
	#
	# `if: always()` so a slow test is still reported when another test failed
	# — the run is red either way, and knowing both is better than knowing one.
	- name: Slow-test budget
	if: always()
	working-directory: dpp-engine
	run: bash scripts/slow-test-check.sh 10 results/*.xml

	debug-prints:
	name: No debug prints in service crates
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	# Forbid println!/eprintln!/dbg! in crates/**/src (service code).
	# The cli/ directory, tests, and benches are explicitly allowed.
	- name: Check for debug prints
	working-directory: dpp-engine
	run: \|
	if grep -rn --include="*.rs" \
	-e '\bprintln!' -e '\beprintln!' -e '\bdbg!' \
	--exclude-dir=tests --exclude-dir=benches \
	crates/dpp-common/src \
	crates/dpp-dal/src \
	crates/dpp-vault/src \
	crates/dpp-identity/src \
	crates/dpp-resolver/src \
	crates/dpp-integrator/src \
	crates/dpp-plugin-host/src \
	crates/dpp-node/src; then
	echo "ERROR: println!/eprintln!/dbg! found in service crate src — use tracing:: instead"
	exit 1
	fi

	subjects-check:
	name: No raw event subject literals outside dpp-common::event
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	# event_type/NATS-subject strings must come from dpp_common::event::subjects,
	# or a renamed subject silently stops matching subscribers.
	- name: Check for raw dpp.passport./dpp.import. literals
	working-directory: dpp-engine
	run: \|
	if grep -rn --include="*.rs" \
	-e '"dpp\.passport\.' -e '"dpp\.import\.' \
	--exclude-dir=tests --exclude-dir=benches \
	--exclude=event.rs \
	crates/dpp-common/src \
	crates/dpp-dal/src \
	crates/dpp-vault/src \
	crates/dpp-identity/src \
	crates/dpp-resolver/src \
	crates/dpp-integrator/src \
	crates/dpp-plugin-host/src \
	crates/dpp-node/src; then
	echo "ERROR: raw dpp.passport./dpp.import. subject literal outside dpp-common::event — use the subjects:: constants"
	exit 1
	fi

	contract-fixture-check:
	name: OpenAPI contract fixtures stay exhaustive
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	# Shell only — no Rust toolchain, so this reports in seconds and is
	# independent of whether the workspace builds. Its own job rather than a
	# step inside another, so "someone disabled the API drift gate" is a
	# distinct red check.
	- name: Fixtures use no struct-update syntax
	working-directory: dpp-engine
	run: bash scripts/contract-fixture-check.sh
	# A grep gate with a broken anchor exits 0 forever and is
	# indistinguishable from a passing one, so the gate is itself tested.
	- name: Gate self-test
	working-directory: dpp-engine
	run: bash scripts/contract-fixture-check.test.sh

	mod-rs-check:
	name: No public items in mod.rs (index files only)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	# mod.rs should be `mod` + `pub use` only. Two allocation-plan
	# exceptions are named and excluded below: service/mod.rs (the
	# PassportService struct + its builders) and validate/mod.rs (the
	# dispatch fn + its error type) — both are the pack's documented
	# deviations, not drift.
	- name: Check for public items in mod.rs
	working-directory: dpp-engine
	run: \|
	exceptions="crates/dpp-vault/src/domain/service/mod.rs crates/dpp-integrator/src/domain/validate/mod.rs"
	violations=""
	for f in $(find crates/*/src cli/src -name mod.rs); do
	skip=false
	for e in $exceptions; do
	[ "$f" = "$e" ] && skip=true
	done
	[ "$skip" = true ] && continue
	if grep -nE '^[[:space:]]*pub[[:space:]]+(struct\|enum\|trait\|fn\|const\|static\|type)\b' "$f" > /dev/null; then
	violations="$violations $f"
	fi
	done
	if [ -n "$violations" ]; then
	echo "ERROR: mod.rs defines public items (should be a pure index) in:$violations"
	exit 1
	fi

	openapi-lint:
	name: OpenAPI spec lints
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	# Node rather than the Rust toolchain, so this is cheap and independent of
	# the build. Redocly 2.x requires Node >= 22.12.
	- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
	with:
	node-version: 24.18.0
	# The description's version and the crate version move together on a
	# release. Checked here as well as in `just check` because this is the
	# job that owns the spec.
	- name: Check the spec version matches the crate version
	working-directory: dpp-engine
	run: bash scripts/spec-version-check.sh
	# The bundles are committed build artifacts. `redocly bundle` is
	# byte-deterministic, so regenerating them and diffing proves the shipped
	# single files still match the multi-file tree they came from.
	#
	# The JSON bundle is the same document in the form the OpenAPI contract
	# test reads (crates/dpp-node/tests/openapi_contract.rs). A stale one
	# would let that test check the types against an outdated spec and pass.
	- name: Check the bundles are current
	working-directory: dpp-engine
	run: \|
	npx --yes @redocly/cli@2.46.2 bundle api/openapi.yaml -o api/openapi.bundled.yaml
	npx --yes @redocly/cli@2.46.2 bundle api/openapi.yaml -o api/openapi.bundled.json
	git diff --exit-code -- api/openapi.bundled.yaml api/openapi.bundled.json \
	\|\| { echo "ERROR: an api/openapi.bundled.* file is stale — run 'just openapi-bundle' and commit both"; exit 1; }
	# The version is pinned and must stay equal to the one in the `just
	# openapi-check` recipe. With `@latest` the two disagree about what is
	# valid the moment Redocly publishes, and a spec that did not change
	# starts failing.
	#
	# Lints the BUNDLE, not the multi-file root: `.redocly.lint-ignore.yaml`
	# baselines today's problems by filename and JSON pointer, and only the
	# bundle keeps the document shape those pointers were written against.
	# Shrink that file, never regenerate it, or the gate stops meaning
	# anything.
	- name: Lint api/openapi.bundled.yaml
	working-directory: dpp-engine
	run: npx --yes @redocly/cli@2.46.2 lint api/openapi.bundled.yaml

	audit:
	name: Security audit
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	- uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2
	with:
	workspaces: dpp-engine
	- name: Install cargo-audit
	run: cargo install cargo-audit --locked
	- name: cargo audit
	working-directory: dpp-engine
	run: cargo audit --deny yanked --deny unmaintained
	- name: Suppression register is current
	working-directory: dpp-engine
	run: bash scripts/check-audit-register.sh
	- name: Suppression register checker self-test
	working-directory: dpp-engine
	run: bash scripts/check-audit-register.test.sh
	- name: Install cargo-deny
	uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2
	with:
	tool: cargo-deny
	- name: cargo deny (bans, licenses, sources)
	working-directory: dpp-engine
	# advisories deliberately excluded — cargo-audit + the register above
	# already own that check; see deny.toml's [advisories] comment.
	run: cargo deny check bans licenses sources

	test-postgres:
	name: Postgres integration tests
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
	with:
	path: dpp-engine
	- uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
	with:
	toolchain: stable
	- uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2
	with:
	workspaces: dpp-engine
	- name: Install cargo-nextest
	uses: taiki-e/install-action@1ed6d7be6168f6c9046541087ff549b6bc581fdf # v2.87.2
	with:
	tool: nextest
	- name: PostgreSQL integration tests (check + clippy + pg_integration)
	working-directory: dpp-engine
	run: \|
	cargo check -p dpp-dal
	cargo check -p dpp-node
	cargo clippy -p dpp-dal -p dpp-node -- -D warnings
	cargo nextest run -p dpp-dal --features integration-tests --no-tests=fail --test pg_integration

Navigation Menu

Uh oh!

perf(dal): upsert scan telemetry in one statement per table (#245) #809

Workflow file

perf(dal): upsert scan telemetry in one statement per table (#245) #809

Uh oh!

Workflow file for this run