manifest.yml

parameters:
  repository: aa8y/postgres-dataset
  platforms: linux/amd64,linux/arm64
templates:
  # Each image carries exactly one dataset. Per-dataset sources/prereqs are
  # declared per tag below and passed to the Dockerfile as build args; optional
  # ones are emitted only when the tag defines them (Mustache sections).
  #
  # Registry build cache (always on, for every tag): import a mode=max build
  # cache from the registry so the EXTRACT layer -- and thus the upstream
  # download -- is reused when nothing relevant changed. The push template below
  # writes the cache, this one only reads it. A missing ref is a cache miss,
  # not an error.
  #
  # The cache lives on GHCR (ghcr.io/<repository>:buildcache-...), not next to
  # the published images on Docker Hub: buildx GETs each cache ref's manifest,
  # and Docker Hub counts every such GET as a pull against the account's
  # metered pull quota (200 per window on a Personal plan) -- ~120 tags x 2
  # arches per CI run blew through it even fully authenticated. GHCR meters
  # nothing for public packages. The packages are public, so a local
  # `dave build` imports anonymously; CI writes them with its GITHUB_TOKEN.
  #
  # Correctness is gated by DATASET_CHECKSUM: bin/dataset-checksum fingerprints
  # the tag's upstream source(s) from cheap metadata (archive.org's JSON
  # metadata API md5/sha1 where available, otherwise HTTP HEAD headers, or the
  # git ls-remote SHA) on the host, and the value is passed as a build arg the
  # builder references just before EXTRACT. When upstream content changes the
  # fingerprint changes, busting EXTRACT -> TRANSFORM -> LOAD; otherwise the
  # cached layers are reused. Computed live via $(...) command substitution, so
  # it stays per-tag without committing anything to this file.
  #
  # Environment knobs. Each is an ordinary shell parameter expansion: dave only
  # renders Mustache, so the literal `${...}` survives into the command and the
  # shell dave spawns it in does the expanding. Every one carries a default that
  # reproduces the previous hardcoded behaviour, so with none of them exported
  # these commands do exactly what they always did. They exist so a single
  # manifest can drive both one runner building everything and a per-arch
  # fan-out that merges afterwards:
  #   PLATFORM        build: restrict the local build to a single platform.
  #                   push:  the one platform to publish. The push template
  #                   re-runs the build (a full local cache hit, so it only
  #                   uploads the GHCR build cache) and then hands the image to
  #                   bin/push-image, which `docker push --platform`es the
  #                   already-built, already-tested image as a single-platform
  #                   manifest. Pushes are PUTs and blob HEADs, which Docker Hub
  #                   does not meter; the previous `buildx build --push` cost
  #                   about one pull per tag re-resolving bases. Multi-platform
  #                   publishing is the merge job's business (bin/merge-manifests),
  #                   never a single machine's.
  #   CACHE_ARCH      suffix for every buildcache ref, so an amd64 and an arm64
  #                   runner write to separate refs (`-amd64`, `-arm64`) instead
  #                   of clobbering one shared one. Unset, the ref is that
  #                   shared legacy ref. (The unsuffixed-ref import fallback
  #                   from before the split is gone: every registry manifest
  #                   fetch counts against Docker Hub's hourly pull quota, and
  #                   one stale-cache lookup per tag per run was costing more
  #                   quota than the cold-cache reuse it might buy. The push
  #                   template imports nothing at all for the same reason -- it
  #                   re-runs a build whose every layer is already in the local
  #                   store, so registry imports there were pure quota burn.)
  #   CACHE_TO_SCOPE  build: additionally EXPORT the cache, under a scoped ref
  #                   (`...-<scope>`). Absent unless set, so an ordinary local
  #                   build still never writes to the registry. CI sets it only
  #                   on a dedicated serialized re-build step, never on the
  #                   parallel build itself: a mode=max export is a heavy
  #                   registry write, and ten shards exporting at build
  #                   concurrency are ~30 simultaneous writers -- the same
  #                   arithmetic that forced `dave push` down to -j 1.
  #   CACHE_FROM_SCOPE  build: additionally IMPORT the scoped ref
  #                   (`...-<scope>`). Set alongside exports of that scope so a
  #                   re-run attempt of a same-repo PR hits the cache its own
  #                   earlier attempt wrote -- without it the retry would
  #                   rebuild, land on a different image ID, and miss the
  #                   integration-test skip stamps keyed on that ID. A ref that
  #                   does not exist yet (the first attempt) is a cache miss,
  #                   not an error.
  #   TAG_SUFFIX      push: suffix for the published tag, so per-arch pushes
  #                   land on `<tag>-amd64` / `<tag>-arm64` for
  #                   bin/merge-manifests to join into one multi-arch tag.
  #   DESCRIPTOR_DIR  push: when set, bin/push-image records the pushed
  #                   manifest's descriptor (digest, size, media type,
  #                   platform) there, one file per ref. bin/merge-manifests
  #                   assembles the multi-arch indexes from those files alone,
  #                   so the merge never reads a manifest back from Docker Hub.
  #
  # `retagFrom: <tag>` marks a tag as a pure alias of another tag in the same
  # context (`latest`, say). There is nothing to build, test or push for an
  # alias -- it is the same image under a second name -- so all three templates
  # are wrapped in an inverted Mustache section on it: for an alias they render
  # to nothing, and dave drops a type whose template rendered empty. The alias
  # is instead materialized registry-side by bin/merge-manifests, which retags
  # the source tag's already-published manifest list. Alias tags also set
  # `structureTest: false`, because container-structure-test needs a local
  # image and no build ever produced one for them.
  build: >
    {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
    --cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
    ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
    ${CACHE_TO_SCOPE:+--cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
    --build-arg DATASET={{dataset}}
    --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
    --build-arg 'EXTRACT_URL={{{extractUrl}}}'
    --build-arg 'SQL_FILES={{{sqlFiles}}}'
    {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
    {{#dbExtension}}--build-arg DB_EXTENSION={{.}}{{/dbExtension}}
    {{#cdDir}}--build-arg 'CD_DIR={{{.}}}'{{/cdDir}}
    {{#indexMethod}}--build-arg INDEX_METHOD={{.}}{{/indexMethod}}
    -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
  # Every engine's smoke test runs through test/integration/with-retry.sh, which
  # re-runs the script once (DDS_ITEST_RETRIES) when it fails in a way that might
  # be infrastructure -- a readiness timeout, a docker or registry hiccup -- and
  # returns immediately on a pass or on the scripts' dedicated assertion exit
  # code, which is deterministic and so pointless to repeat. See docs/testing.md.
  test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run.sh {{tag}} {{dataset}}{{/retagFrom}}'
  push: >
    {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
    --cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
    --build-arg DATASET={{dataset}}
    --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
    --build-arg 'EXTRACT_URL={{{extractUrl}}}'
    --build-arg 'SQL_FILES={{{sqlFiles}}}'
    {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
    {{#dbExtension}}--build-arg DB_EXTENSION={{.}}{{/dbExtension}}
    {{#cdDir}}--build-arg 'CD_DIR={{{.}}}'{{/cdDir}}
    {{#indexMethod}}--build-arg INDEX_METHOD={{.}}{{/indexMethod}}
    -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
contexts:
  postgres:
    tagKeys: dataset
    # Shared assertions for every postgres tag (the POSTGRES_* env wiring and
    # init-script dir). Scoped to this context -- structureTest configs trickle
    # down and concatenate, so a global block would wrongly apply to mysql too.
    structureTest:
      configs:
        - test/config/common.yaml
    tags:
      airlines:
        extractUrl: https://edu.postgrespro.ru/demo-20250901-3m.sql.gz
        sqlFiles: airlines.sql
        structureTest:
          configs:
            - test/config/airlines.yaml
      adventureworks:
        extractUrl: https://github.com/lorint/AdventureWorks-for-Postgres.git
        sqlFiles: AdventureWorks-for-Postgres/install.sql
        extraPrereqs: python3
        cdDir: /docker-entrypoint-initdb.d/adventureworks
        structureTest:
          configs:
            - test/config/adventureworks.yaml
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        structureTest:
          configs:
            - test/config/dellstore.yaml
      # Employees: the datacharmer/test_db large sample (300,024 employees,
      # 2,844,047 salary rows), from the repo's own upstream-maintained
      # PostgreSQL port. Upstream splits it into a schema file plus eight data
      # dumps loaded by a shell script, so the scripts/employees transform hook
      # assembles them into one init script (backticks stripped, the port's
      # DROP/CREATE DATABASE header removed). The dataset is fixed, so the smoke
      # test records exact counts, identical to the mysql employees tag.
      employees:
        extractUrl: https://github.com/datacharmer/test_db.git
        sqlFiles: employees.sql
        structureTest:
          configs:
            - test/config/employees.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        structureTest:
          configs:
            - test/config/frenchtowns.yaml
      # GeoNames cities15000. Published only as a zipped tab-separated export, so
      # the schema is authored in-repo (postgres/scripts/geonames) and the export
      # ships alongside the init script for \copy to read at start. Upstream
      # refreshes the dump daily, so the smoke test records floors.
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        sqlFiles: geonames.sql
        cdDir: /docker-entrypoint-initdb.d/geonames
        structureTest:
          configs:
            - test/config/geonames.yaml
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        structureTest:
          configs:
            - test/config/iso3166.yaml
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        sqlFiles: moma.sql
        cdDir: /docker-entrypoint-initdb.d/moma
        structureTest:
          configs:
            - test/config/moma.yaml
      omdb:
        extractUrl: https://github.com/df7cb/omdb-postgresql.git
        sqlFiles: >-
          omdb-postgresql/00-drop-tables.sql
          omdb-postgresql/12-kind.sql
          omdb-postgresql/13-schema.sql
          omdb-postgresql/20-import.sql
          omdb-postgresql/30-duplicates.sql
          omdb-postgresql/31-primary-keys.sql
          omdb-postgresql/40-clean-orphans.sql
          omdb-postgresql/41-foreign-keys.sql
          omdb-postgresql/50-views.sql
          omdb-postgresql/52-category-en.sql
          omdb-postgresql/55-purge_dirty.sql
          omdb-postgresql/60-log.sql
        dbExtension: tsm_system_rows
        cdDir: /docker-entrypoint-initdb.d
        structureTest:
          configs:
            - test/config/omdb.yaml
      # OpenFlights airports/airlines/routes. Published only as three
      # comma-separated data files, so the schema is authored in-repo
      # (postgres/scripts/openflights) and the files ship alongside the init
      # script for \copy to read at start. Upstream refreshes them in place, so
      # the smoke test records floors.
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        sqlFiles: openflights.sql
        cdDir: /docker-entrypoint-initdb.d/openflights
        structureTest:
          configs:
            - test/config/openflights.yaml
      pagila:
        # A pinned tarball, not a clone of upstream git HEAD: pagila's dump
        # changes shape over time (2026-07 added a pgvector example, which is
        # why the image now ships pgvector -- see postgres/Dockerfile), and the
        # recorded expected counts are measured against one specific commit.
        # Bump the pin deliberately: re-record expectations with
        # `test/integration/run.sh --update` and revisit the structure-test
        # config, alongside any new extension the dump starts requiring.
        extractUrl: https://github.com/devrimgunduz/pagila/archive/eddcfc4513dab4239ee1acbdf52beb1f5fcf50ac.tar.gz
        sqlFiles: pagila-eddcfc4513dab4239ee1acbdf52beb1f5fcf50ac/pagila-schema.sql pagila-eddcfc4513dab4239ee1acbdf52beb1f5fcf50ac/pagila-data.sql
        structureTest:
          configs:
            - test/config/pagila.yaml
      # StackExchange sites all share one schema, so they all build through the
      # shared scripts/stackexchange hooks (each site's scripts/<site> dir is a
      # symlink to it). Adding a site is just a tag here + a structure-test
      # config + an expected/<site>.json floor file.
      stackexchange-beer:
        dataset: beer
        extractUrl: https://archive.org/download/stackexchange/beer.stackexchange.com.7z
        sqlFiles: beer.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-beer.yaml
      stackexchange-coffee:
        dataset: coffee
        extractUrl: https://archive.org/download/stackexchange/coffee.stackexchange.com.7z
        sqlFiles: coffee.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-coffee.yaml
      stackexchange-poker:
        dataset: poker
        extractUrl: https://archive.org/download/stackexchange/poker.stackexchange.com.7z
        sqlFiles: poker.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-poker.yaml
      stackexchange-woodworking:
        dataset: woodworking
        extractUrl: https://archive.org/download/stackexchange/woodworking.stackexchange.com.7z
        sqlFiles: woodworking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-woodworking.yaml
      stackexchange-chess:
        dataset: chess
        extractUrl: https://archive.org/download/stackexchange/chess.stackexchange.com.7z
        sqlFiles: chess.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-chess.yaml
      stackexchange-cooking:
        dataset: cooking
        extractUrl: https://archive.org/download/stackexchange/cooking.stackexchange.com.7z
        sqlFiles: cooking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-cooking.yaml
      stackexchange-outdoors:
        dataset: outdoors
        extractUrl: https://archive.org/download/stackexchange/outdoors.stackexchange.com.7z
        sqlFiles: outdoors.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-outdoors.yaml
      stackexchange-boardgames:
        dataset: boardgames
        extractUrl: https://archive.org/download/stackexchange/boardgames.stackexchange.com.7z
        sqlFiles: boardgames.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/stackexchange-boardgames.yaml
      sportsdb:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_tables.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_inserts.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_indexes.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_constraints.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_fks.sql
        sqlFiles: sportsdb_tables.sql sportsdb_inserts.sql sportsdb_indexes.sql sportsdb_constraints.sql sportsdb_fks.sql
        indexMethod: btree
        structureTest:
          configs:
            - test/config/sportsdb.yaml
      chinook:
        dataset: chinook
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_genres_artists_albums.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_songs.sql
        sqlFiles: chinook_ddl.sql chinook_genres_artists_albums.sql chinook_songs.sql
        structureTest:
          configs:
            - test/config/chinook.yaml
      northwind:
        dataset: northwind
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/northwind_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/northwind_data.sql
        sqlFiles: northwind_ddl.sql northwind_data.sql
        structureTest:
          configs:
            - test/config/northwind.yaml
      pgexercises:
        dataset: pgexercises
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        structureTest:
          configs:
            - test/config/pgexercises.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        structureTest:
          configs:
            - test/config/usda.yaml
      world:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/world/world-1.0/world-1.0.tar.gz
        sqlFiles: dbsamples-0.1/world/world.sql
        structureTest:
          configs:
            - test/config/world.yaml
      # `all` is retired (multi-dataset images are legacy). `latest` now tracks a
      # single representative dataset -- as an alias of it, not a second build.
      latest:
        retagFrom: world
        structureTest: false
  # MySQL images. Built on a thin Alpine MariaDB base (there is no official
  # Alpine MySQL image; MariaDB is the MySQL drop-in Alpine itself ships) and
  # published under a separate repository. The build/test/push templates are
  # the MySQL flavours (fewer build args than postgres; test runs the MySQL
  # smoke-test script) and override the global postgres ones for this context.
  mysql:
    tagKeys: dataset
    parameters:
      repository: aa8y/mysql-dataset
    structureTest:
      configs:
        - test/config/mysql/common.yaml
    templates:
      # Mirrors the global templates (see top of file), including the always-on
      # registry build cache gated by the DATASET_CHECKSUM fingerprint.
      build: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
        ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
        ${CACHE_TO_SCOPE:+--cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
      test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run-mysql.sh {{tag}} {{dataset}}{{/retagFrom}}'
      push: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
    tags:
      # Sakila is MySQL's own sample database (the original of the model pagila
      # ports to Postgres). Ships schema + data as separate files in a tarball.
      sakila:
        extractUrl: https://downloads.mysql.com/docs/sakila-db.tar.gz
        sqlFiles: sakila-db/sakila-schema.sql sakila-db/sakila-data.sql
        structureTest:
          configs:
            - test/config/mysql/sakila.yaml
      # MySQL's canonical `world` sample (city, country, countrylanguage) -- the
      # same three tables as the Postgres `world` dataset.
      world:
        extractUrl: https://downloads.mysql.com/docs/world-db.tar.gz
        sqlFiles: world-db/world.sql
        structureTest:
          configs:
            - test/config/mysql/world.yaml
      # Chinook digital-media store. The vendor ships a MySQL-specific script as
      # a GitHub release asset; the build strips its `CREATE DATABASE Chinook` /
      # `USE` so it loads into the lowercase `chinook` database.
      chinook:
        extractUrl: https://github.com/lerocha/chinook-database/releases/download/v1.4.5/Chinook_MySql.sql
        sqlFiles: Chinook_MySql.sql
        structureTest:
          configs:
            - test/config/mysql/chinook.yaml
      # Northwind: the classic Microsoft small-business sample, ported to MySQL
      # by dalers/mywind. The build strips its `CREATE SCHEMA northwind` / `USE`
      # (the table DDL is schema-qualified to `northwind`, which the header
      # creates).
      northwind:
        extractUrl: https://github.com/dalers/mywind.git
        sqlFiles: mywind/northwind.sql mywind/northwind-data.sql
        structureTest:
          configs:
            - test/config/mysql/northwind.yaml
      # Employees: MySQL's canonical large sample (300k employees, 2.8M salary
      # rows). Upstream ships a loader script that pulls in six data files with
      # the mysql client's `source` command, so the scripts/employees transform
      # hook inlines each of them into a single self-contained init script.
      employees:
        extractUrl: https://github.com/datacharmer/test_db.git
        sqlFiles: test_db/employees.sql
        structureTest:
          configs:
            - test/config/mysql/employees.yaml
      # MoMA research collection. Published only as CSV, so the schema is
      # authored in-repo (mysql/scripts/moma) and the CSVs are bulk-loaded at
      # start with server-side LOAD DATA INFILE. Counts drift between builds (the
      # CSVs are refreshed upstream), so the smoke test records floors.
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        sqlFiles: moma.sql
        structureTest:
          configs:
            - test/config/mysql/moma.yaml
      # GeoNames cities15000. Published only as a zipped tab-separated export, so
      # the schema is authored in-repo (mysql/scripts/geonames) and the export is
      # bulk-loaded at start with server-side LOAD DATA INFILE. Upstream
      # refreshes the dump daily, so the smoke test records floors.
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        sqlFiles: geonames.sql
        structureTest:
          configs:
            - test/config/mysql/geonames.yaml
      # OpenFlights airports/airlines/routes. Published only as three
      # comma-separated data files, so the schema is authored in-repo
      # (mysql/scripts/openflights) and the files are bulk-loaded at start with
      # server-side LOAD DATA INFILE. Upstream refreshes them in place, so the
      # smoke test records floors.
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        sqlFiles: openflights.sql
        structureTest:
          configs:
            - test/config/mysql/openflights.yaml
      # Tier 1 hand-translated PostgreSQL datasets. These ship only as
      # PostgreSQL dumps, but are plain DDL + data (no PL/pgSQL, no extensions),
      # so the shared scripts/pgsql transform hook rewrites the dialect at build
      # time (COPY -> INSERT, type/identifier/sequence fixes). Each dataset's
      # scripts/<dataset> dir is a symlink to scripts/pgsql; only python3 is
      # needed beyond the base tools.
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/mysql/iso3166.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/mysql/frenchtowns.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/mysql/usda.yaml
      pgexercises:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/mysql/pgexercises.yaml
      # Tier 2 hand-translated PostgreSQL datasets. Larger / slightly more
      # involved than tier 1 but still handled by the shared scripts/pgsql hook:
      # sportsdb is INSERT-based across five files with sequences, constraints,
      # FKs and Yugabyte `USING lsm` indexes; dellstore2 is COPY-based plus a
      # PL/pgSQL stored function, which the hook drops (no mechanical MySQL
      # translation -- the schema and all data still load faithfully).
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/mysql/dellstore.yaml
      sportsdb:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_tables.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_inserts.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_indexes.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_constraints.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_fks.sql
        sqlFiles: sportsdb_tables.sql sportsdb_inserts.sql sportsdb_indexes.sql sportsdb_constraints.sql sportsdb_fks.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/mysql/sportsdb.yaml
      # StackExchange sites. Each ships as per-table XML in a 7z archive (no
      # SQL), so -- exactly like the postgres side -- every site builds through
      # the shared scripts/stackexchange hooks (each site's scripts/<site> dir
      # is a symlink to it): the extract hook un-7z's the dump and the transform
      # hook converts the XML to a MySQL init script (CREATE TABLE + batched
      # INSERTs + indexes). The transform is the MySQL-emitting counterpart of
      # the postgres one; adding a site is just a tag here + a structure-test
      # config + an expected/mysql/<site>.json floor file.
      stackexchange-beer:
        dataset: beer
        extractUrl: https://archive.org/download/stackexchange/beer.stackexchange.com.7z
        sqlFiles: beer.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-beer.yaml
      stackexchange-coffee:
        dataset: coffee
        extractUrl: https://archive.org/download/stackexchange/coffee.stackexchange.com.7z
        sqlFiles: coffee.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-coffee.yaml
      stackexchange-poker:
        dataset: poker
        extractUrl: https://archive.org/download/stackexchange/poker.stackexchange.com.7z
        sqlFiles: poker.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-poker.yaml
      stackexchange-woodworking:
        dataset: woodworking
        extractUrl: https://archive.org/download/stackexchange/woodworking.stackexchange.com.7z
        sqlFiles: woodworking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-woodworking.yaml
      stackexchange-chess:
        dataset: chess
        extractUrl: https://archive.org/download/stackexchange/chess.stackexchange.com.7z
        sqlFiles: chess.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-chess.yaml
      stackexchange-cooking:
        dataset: cooking
        extractUrl: https://archive.org/download/stackexchange/cooking.stackexchange.com.7z
        sqlFiles: cooking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-cooking.yaml
      stackexchange-outdoors:
        dataset: outdoors
        extractUrl: https://archive.org/download/stackexchange/outdoors.stackexchange.com.7z
        sqlFiles: outdoors.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-outdoors.yaml
      stackexchange-boardgames:
        dataset: boardgames
        extractUrl: https://archive.org/download/stackexchange/boardgames.stackexchange.com.7z
        sqlFiles: boardgames.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/mysql/stackexchange-boardgames.yaml
      # `latest` tracks a single representative dataset, mirroring postgres.
      latest:
        retagFrom: world
        structureTest: false
  # CockroachDB images. Built on the official cockroachdb/cockroach image and
  # published under a separate repository. There is no official Alpine image,
  # but the official one is slim (UBI-minimal, ~170 MB) and multi-arch, and its
  # entrypoint honours the same /docker-entrypoint-initdb.d/*.sql convention the
  # official postgres image uses (when started with `start-single-node`). Since
  # CockroachDB is PostgreSQL wire- and SQL-compatible, these reuse the same
  # PostgreSQL-dialect sample dumps the Yugabyte postgres tags do. The build
  # templates are the CockroachDB flavours (fewer build args than postgres; test
  # runs the CockroachDB smoke-test script) and override the global postgres
  # ones for this context.
  cockroach:
    tagKeys: dataset
    parameters:
      repository: aa8y/cockroach-dataset
    structureTest:
      configs:
        - test/config/cockroach/common.yaml
    templates:
      # Mirrors the global templates (see top of file), including the always-on
      # registry build cache gated by the DATASET_CHECKSUM fingerprint.
      build: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
        ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
        ${CACHE_TO_SCOPE:+--cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        {{#cdDir}}--build-arg 'CD_DIR={{{.}}}'{{/cdDir}}
        {{#indexMethod}}--build-arg INDEX_METHOD={{.}}{{/indexMethod}}
        -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
      test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run-cockroach.sh {{tag}} {{dataset}}{{/retagFrom}}'
      push: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        {{#cdDir}}--build-arg 'CD_DIR={{{.}}}'{{/cdDir}}
        {{#indexMethod}}--build-arg INDEX_METHOD={{.}}{{/indexMethod}}
        -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
    tags:
      # Chinook digital-media store, sourced from Yugabyte's PostgreSQL-dialect
      # mirror (the same dump behind the postgres `chinook` tag): plain
      # DDL + INSERT data with quoted CamelCase identifiers, which load on
      # CockroachDB unchanged.
      chinook:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_genres_artists_albums.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_songs.sql
        sqlFiles: chinook_ddl.sql chinook_genres_artists_albums.sql chinook_songs.sql
        # python3 for the scripts/chinook -> yugabyte transform (coalesces the
        # dump's single-row INSERTs into batched multi-row INSERTs).
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/chinook.yaml
      # Northwind specialty-foods sample, same Yugabyte PostgreSQL-dialect dump
      # behind the postgres `northwind` tag (DDL + INSERT data).
      northwind:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/northwind_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/northwind_data.sql
        sqlFiles: northwind_ddl.sql northwind_data.sql
        structureTest:
          configs:
            - test/config/cockroach/northwind.yaml
      # Sakila, the MySQL DVD-rental sample (the original of pagila), from
      # jOOQ's multi-dialect port collection, which ships a CockroachDB-specific
      # flavour of the PostgreSQL dump: no `film_text` table (15 base tables,
      # where MySQL has 16), `release_year` as a plain integer rather than the
      # PostgreSQL `year` domain, and a `payment` table without the inherited
      # per-month partitions. The repo is cloned (like the mysql `northwind`
      # tag) but the data is fixed, so counts are recorded exactly.
      #
      # Data loads through the shared scripts/pgfoundry hook (scripts/sakila is
      # a symlink to it): the schema is CockroachDB-clean as published -- ENUM
      # type, `text[]`, `bytea`, four views, `SET search_path`/`default_tablespace`
      # and `ALTER ... OWNER TO root` all run unchanged on v25.4 -- but the
      # 46,273 data rows arrive in 15 `COPY ... FROM stdin` blocks, and CRDB's
      # init-time stdin COPY is far slower than batched INSERTs. The hook also
      # drops the dump's trailing per-table `analyze`. The staff<->store foreign
      # key cycle the data file breaks at the top and restores at the end is
      # kept as published.
      sakila:
        extractUrl: https://github.com/jOOQ/sakila.git
        sqlFiles: sakila/cockroachdb-sakila-db/cockroachdb-sakila-schema.sql sakila/cockroachdb-sakila-db/cockroachdb-sakila-insert-data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/sakila.yaml
      # Plain PostgreSQL-dialect dumps from pgFoundry or Yugabyte that load on
      # CockroachDB unchanged (mirroring the postgres tags for these datasets).
      world:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/world/world-1.0/world-1.0.tar.gz
        sqlFiles: dbsamples-0.1/world/world.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/world.yaml
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/iso3166.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/frenchtowns.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/usda.yaml
      pgexercises:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        structureTest:
          configs:
            - test/config/cockroach/pgexercises.yaml
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/dellstore.yaml
      sportsdb:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_tables.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_inserts.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_indexes.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_constraints.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_fks.sql
        sqlFiles: sportsdb_tables.sql sportsdb_inserts.sql sportsdb_indexes.sql sportsdb_constraints.sql sportsdb_fks.sql
        extraPrereqs: python3
        indexMethod: btree
        structureTest:
          configs:
            - test/config/cockroach/sportsdb.yaml
      # MoMA publishes only CSV, so the schema is authored in-repo
      # (cockroach/scripts/moma) and the transform hook bakes the CSV data into
      # the init script as INSERTs (CockroachDB's SQL client supports neither
      # \copy nor COPY FROM '<file>'), so nothing ships beside the init script.
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        sqlFiles: moma.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/moma.yaml
      # GeoNames cities15000. Published only as a zipped tab-separated export, so
      # the schema is authored in-repo (cockroach/scripts/geonames) and the
      # export is bulk-loaded with IMPORT INTO over its DELIMITED (TSV) format.
      # Upstream refreshes the dump daily, so the smoke test records floors.
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        sqlFiles: geonames.sql
        structureTest:
          configs:
            - test/config/cockroach/geonames.yaml
      # OpenFlights airports/airlines/routes. Published only as three
      # comma-separated data files, so the schema is authored in-repo
      # (cockroach/scripts/openflights) and the files are bulk-loaded with
      # IMPORT INTO straight from the node's external-IO dir (they are already
      # RFC4180 CSV, so unlike moma they need no normalizing). Upstream
      # refreshes them in place, so the smoke test records floors.
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        sqlFiles: openflights.sql
        structureTest:
          configs:
            - test/config/cockroach/openflights.yaml
      # Employees: the datacharmer/test_db large sample (300,024 employees,
      # 2,844,047 salary rows). CockroachDB runs upstream's PostgreSQL port
      # unchanged apart from its DROP/CREATE DATABASE preamble, but 4.2M rows of
      # init-time INSERTs are hopeless here (every batch is a Raft commit), so
      # the scripts/employees hooks (python3) rewrite the eight data dumps into
      # per-table CSVs, ship them to the node's external-IO dir and load them
      # with IMPORT INTO -- the same bulk path geonames and openflights use.
      # The dataset is fixed, so the smoke test records exact counts, identical
      # to the mysql, postgres, sqlite and duckdb employees tags.
      employees:
        extractUrl: https://github.com/datacharmer/test_db.git
        sqlFiles: employees.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/cockroach/employees.yaml
      # StackExchange sites all share one schema, so they all build through the
      # shared scripts/stackexchange hooks (each site's scripts/<site> dir is a
      # symlink to it). Adding a site is just a tag here + a structure-test
      # config + an expected/cockroach/<site>.json floor file.
      stackexchange-beer:
        dataset: beer
        extractUrl: https://archive.org/download/stackexchange/beer.stackexchange.com.7z
        sqlFiles: beer.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-beer.yaml
      stackexchange-coffee:
        dataset: coffee
        extractUrl: https://archive.org/download/stackexchange/coffee.stackexchange.com.7z
        sqlFiles: coffee.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-coffee.yaml
      stackexchange-poker:
        dataset: poker
        extractUrl: https://archive.org/download/stackexchange/poker.stackexchange.com.7z
        sqlFiles: poker.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-poker.yaml
      stackexchange-woodworking:
        dataset: woodworking
        extractUrl: https://archive.org/download/stackexchange/woodworking.stackexchange.com.7z
        sqlFiles: woodworking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-woodworking.yaml
      stackexchange-chess:
        dataset: chess
        extractUrl: https://archive.org/download/stackexchange/chess.stackexchange.com.7z
        sqlFiles: chess.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-chess.yaml
      stackexchange-cooking:
        dataset: cooking
        extractUrl: https://archive.org/download/stackexchange/cooking.stackexchange.com.7z
        sqlFiles: cooking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-cooking.yaml
      stackexchange-outdoors:
        dataset: outdoors
        extractUrl: https://archive.org/download/stackexchange/outdoors.stackexchange.com.7z
        sqlFiles: outdoors.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-outdoors.yaml
      stackexchange-boardgames:
        dataset: boardgames
        extractUrl: https://archive.org/download/stackexchange/boardgames.stackexchange.com.7z
        sqlFiles: boardgames.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/cockroach/stackexchange-boardgames.yaml
      # `latest` tracks a single representative dataset, mirroring postgres/mysql.
      latest:
        retagFrom: chinook
        structureTest: false
  # SQLite images. Built on the Alpine, statically-linked keinos/sqlite3 image
  # and published under a separate repository. SQLite is serverless -- a
  # database is just a file -- so the build *builds the database file* rather
  # than shipping init scripts: a dataset is described either by a text SQL
  # source (sqlFiles, fed to the sqlite3 CLI to build the database) or by a
  # prebuilt SQLite database file (dbFile, shipped as-is). The build templates
  # are the SQLite flavours (no init-script assembly; test runs the SQLite
  # smoke-test script) and override the global postgres ones for this context.
  sqlite:
    tagKeys: dataset
    parameters:
      repository: aa8y/sqlite-dataset
    structureTest:
      configs:
        - test/config/sqlite/common.yaml
    templates:
      # Mirrors the global templates (see top of file), including the always-on
      # registry build cache gated by the DATASET_CHECKSUM fingerprint. Emits
      # SQL_FILES or DB_FILE per tag (Mustache sections), never both.
      build: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
        ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
        ${CACHE_TO_SCOPE:+--cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        {{#sqlFiles}}--build-arg 'SQL_FILES={{{.}}}'{{/sqlFiles}}
        {{#dbFile}}--build-arg 'DB_FILE={{{.}}}'{{/dbFile}}
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
      test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run-sqlite.sh {{tag}} {{dataset}}{{/retagFrom}}'
      push: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        {{#sqlFiles}}--build-arg 'SQL_FILES={{{.}}}'{{/sqlFiles}}
        {{#dbFile}}--build-arg 'DB_FILE={{{.}}}'{{/dbFile}}
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
    tags:
      # Chinook digital-media store. The vendor ships a native SQLite SQL script
      # (release v1.4.5), which the build runs through the sqlite3 CLI to create
      # /data/chinook.db. Same CamelCase identifiers as the other chinook tags.
      chinook:
        extractUrl: https://github.com/lerocha/chinook-database/releases/download/v1.4.5/Chinook_Sqlite.sql
        sqlFiles: Chinook_Sqlite.sql
        structureTest:
          configs:
            - test/config/sqlite/chinook.yaml
      # Northwind specialty-foods sample. The jpwhite3/northwind-SQLite3 port
      # publishes a prebuilt database (dist/northwind.db), shipped as-is. This is
      # the expanded edition of the sample (a large Order Details table).
      northwind:
        extractUrl: https://raw.githubusercontent.com/jpwhite3/northwind-SQLite3/main/dist/northwind.db
        dbFile: northwind.db
        structureTest:
          configs:
            - test/config/sqlite/northwind.yaml
      # Tier 1 hand-translated PostgreSQL datasets. These ship only as
      # PostgreSQL dumps, but are plain DDL + data (no PL/pgSQL, no extensions),
      # so the shared scripts/pgsql transform hook rewrites the dialect at build
      # time (COPY -> INSERT, strip Postgres-only noise). SQLite cannot add
      # constraints via ALTER TABLE, so PK/FK/unique from the dump are dropped;
      # tables and row counts still load faithfully. Each dataset's
      # scripts/<dataset> dir is a symlink to scripts/pgsql.
      world:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/world/world-1.0/world-1.0.tar.gz
        sqlFiles: dbsamples-0.1/world/world.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/world.yaml
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/iso3166.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/frenchtowns.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/usda.yaml
      pgexercises:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/pgexercises.yaml
      # Tier 2 hand-translated PostgreSQL datasets. Larger / slightly more
      # involved than tier 1 but still handled by the shared scripts/pgsql hook:
      # sportsdb is INSERT-based across five files with sequences and constraints;
      # dellstore2 is COPY-based plus a PL/pgSQL stored function, which the hook
      # drops (no mechanical SQLite translation -- the schema and all data still
      # load faithfully).
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/dellstore.yaml
      sportsdb:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_tables.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_inserts.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_indexes.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_constraints.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_fks.sql
        sqlFiles: sportsdb_tables.sql sportsdb_inserts.sql sportsdb_indexes.sql sportsdb_constraints.sql sportsdb_fks.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/sportsdb.yaml
      # Airlines: the postgrespro flight-bookings demo (same pinned snapshot as
      # the postgres tag), shipped as a single gzipped pg_dump that manages its
      # own `demo` database and qualifies everything to a `bookings` schema.
      # The scripts/airlines hooks decompress the dump, then enable the shared
      # scripts/pgsql transform's opt-in knobs (strip the `bookings.`
      # qualifier, drop the three convenience views) and delegate to it; jsonb
      # and range/interval columns load as plain text via type affinity.
      airlines:
        extractUrl: https://edu.postgrespro.ru/demo-20250901-3m.sql.gz
        sqlFiles: airlines.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/sqlite/airlines.yaml
      # Employees: the datacharmer/test_db large sample (300,024 employees,
      # 2,844,047 salary rows -- by far the heaviest sqlite image). Upstream
      # ships no SQLite port, but its PostgreSQL one is near dialect-free, so
      # the scripts/employees transform hook uses that schema with three
      # rewrites SQLite forces (no DB management, no multi-table DROP TABLE, no
      # `CREATE OR REPLACE VIEW`) and appends the eight data dumps unchanged,
      # backticks stripped, inside one transaction. The dataset is fixed, so
      # the smoke test records exact counts, identical to the mysql and
      # postgres employees tags.
      employees:
        extractUrl: https://github.com/datacharmer/test_db.git
        sqlFiles: employees.sql
        structureTest:
          configs:
            - test/config/sqlite/employees.yaml
      # Sakila: MySQL's own sample database (the original of pagila). The
      # bradleygrant/sakila-sqlite3 port publishes a prebuilt database file,
      # shipped as-is (mirrors the northwind dbFile pattern).
      sakila:
        extractUrl: https://raw.githubusercontent.com/bradleygrant/sakila-sqlite3/main/sakila_master.db
        dbFile: sakila_master.db
        structureTest:
          configs:
            - test/config/sqlite/sakila.yaml
      # MoMA research collection. Published only as CSV, so the schema is
      # authored in-repo (sqlite/scripts/moma) and the CSVs are bulk-loaded at
      # build time with the sqlite3 CLI's .import dot-command. Counts drift
      # between builds (the CSVs are refreshed upstream), so the smoke test
      # records floors.
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        sqlFiles: moma.sql
        structureTest:
          configs:
            - test/config/sqlite/moma.yaml
      # GeoNames cities15000. Published only as a zipped tab-separated export, so
      # the schema is authored in-repo (sqlite/scripts/geonames) and the export
      # is bulk-loaded at build time with the sqlite3 CLI's .import dot-command.
      # Upstream refreshes the dump daily, so the smoke test records floors.
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        sqlFiles: geonames.sql
        structureTest:
          configs:
            - test/config/sqlite/geonames.yaml
      # OpenFlights airports/airlines/routes. Published only as three
      # comma-separated data files, so the schema is authored in-repo
      # (sqlite/scripts/openflights) and the files are bulk-loaded at build time
      # with the sqlite3 CLI's .import dot-command. Upstream refreshes them in
      # place, so the smoke test records floors.
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        sqlFiles: openflights.sql
        structureTest:
          configs:
            - test/config/sqlite/openflights.yaml
      # StackExchange sites. Each ships as per-table XML in a 7z archive (no
      # SQL), so -- exactly like the postgres/mysql side -- every site builds
      # through the shared scripts/stackexchange hooks (each site's scripts/<site>
      # dir is a symlink to it): the extract hook un-7z's the dump and the
      # transform hook converts the XML to a SQLite init script (CREATE TABLE +
      # batched INSERTs + indexes). Adding a site is just a tag here + a
      # structure-test config + an expected/sqlite/<site>.json floor file.
      stackexchange-beer:
        dataset: beer
        extractUrl: https://archive.org/download/stackexchange/beer.stackexchange.com.7z
        sqlFiles: beer.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-beer.yaml
      stackexchange-coffee:
        dataset: coffee
        extractUrl: https://archive.org/download/stackexchange/coffee.stackexchange.com.7z
        sqlFiles: coffee.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-coffee.yaml
      stackexchange-poker:
        dataset: poker
        extractUrl: https://archive.org/download/stackexchange/poker.stackexchange.com.7z
        sqlFiles: poker.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-poker.yaml
      stackexchange-woodworking:
        dataset: woodworking
        extractUrl: https://archive.org/download/stackexchange/woodworking.stackexchange.com.7z
        sqlFiles: woodworking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-woodworking.yaml
      stackexchange-chess:
        dataset: chess
        extractUrl: https://archive.org/download/stackexchange/chess.stackexchange.com.7z
        sqlFiles: chess.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-chess.yaml
      stackexchange-cooking:
        dataset: cooking
        extractUrl: https://archive.org/download/stackexchange/cooking.stackexchange.com.7z
        sqlFiles: cooking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-cooking.yaml
      stackexchange-outdoors:
        dataset: outdoors
        extractUrl: https://archive.org/download/stackexchange/outdoors.stackexchange.com.7z
        sqlFiles: outdoors.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-outdoors.yaml
      stackexchange-boardgames:
        dataset: boardgames
        extractUrl: https://archive.org/download/stackexchange/boardgames.stackexchange.com.7z
        sqlFiles: boardgames.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/sqlite/stackexchange-boardgames.yaml
      # `latest` tracks a single representative dataset, mirroring postgres/mysql.
      latest:
        retagFrom: chinook
        structureTest: false
  # DuckDB images. Built on the official duckdb/duckdb image and published
  # under a separate repository. DuckDB is embedded like SQLite -- a database
  # is just a file -- so the build *builds the database file*: a dataset is
  # described either by a text SQL source (sqlFiles, fed to the duckdb CLI) or
  # by a prebuilt DuckDB database file (dbFile, shipped as-is). The official
  # image is distroless (no shell), so the Dockerfile adds a glibc loader
  # stage between the Alpine builder and the final image; see
  # duckdb/Dockerfile for the full shape. The build templates are the DuckDB
  # flavours (same arg surface as sqlite; test runs the DuckDB smoke-test
  # script) and override the global postgres ones for this context.
  duckdb:
    tagKeys: dataset
    parameters:
      repository: aa8y/duckdb-dataset
    structureTest:
      configs:
        - test/config/duckdb/common.yaml
    templates:
      # Mirrors the global templates (see top of file), including the always-on
      # registry build cache gated by the DATASET_CHECKSUM fingerprint. Emits
      # SQL_FILES or DB_FILE per tag (Mustache sections), never both.
      build: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
        ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
        ${CACHE_TO_SCOPE:+--cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        {{#sqlFiles}}--build-arg 'SQL_FILES={{{.}}}'{{/sqlFiles}}
        {{#dbFile}}--build-arg 'DB_FILE={{{.}}}'{{/dbFile}}
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
      test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run-duckdb.sh {{tag}} {{dataset}}{{/retagFrom}}'
      push: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-to type=registry,ref=ghcr.io/{{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        {{#sqlFiles}}--build-arg 'SQL_FILES={{{.}}}'{{/sqlFiles}}
        {{#dbFile}}--build-arg 'DB_FILE={{{.}}}'{{/dbFile}}
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
    tags:
      # Chinook digital-media store, from the same pinned vendor release the
      # sqlite chinook tag uses. The scripts/chinook transform hook (python3)
      # rewrites the script's two SQLite-isms -- bracket-quoted identifiers and
      # forward-referencing FOREIGN KEYs -- into DuckDB SQL; row counts match
      # the other chinook tags exactly.
      chinook:
        extractUrl: https://github.com/lerocha/chinook-database/releases/download/v1.4.5/Chinook_Sqlite.sql
        sqlFiles: Chinook_Sqlite.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/chinook.yaml
      # Employees: the datacharmer/test_db large sample (300,024 employees,
      # 2,844,047 salary rows). Upstream ships no DuckDB port, so the
      # scripts/employees transform hook (python3) takes its PostgreSQL one --
      # DuckDB runs that schema nearly as written -- drops the constructs DuckDB
      # rejects (multi-table DROP TABLE, `FOREIGN KEY ... ON DELETE CASCADE`)
      # and rewrites the eight data dumps into per-table CSVs the loader reads
      # with COPY, which is far faster and lighter than parsing 25k-tuple
      # INSERTs. The dataset is fixed, so the smoke test records exact counts,
      # identical to the mysql, postgres and sqlite employees tags.
      employees:
        extractUrl: https://github.com/datacharmer/test_db.git
        sqlFiles: employees.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/employees.yaml
      # PostgreSQL-dump datasets, the same sources the sqlite tags use. DuckDB's
      # parser is PostgreSQL-derived, so the shared scripts/pgsql transform hook
      # is a thin one next to SQLite's: it converts COPY blocks to INSERTs,
      # transcodes Latin-1 dumps, and removes only what DuckDB genuinely cannot
      # run (SET, setval, GRANT, USING btree, serial, FOREIGN KEYs, PL/pgSQL) --
      # PostgreSQL casts, column types and `ALTER TABLE ... ADD PRIMARY KEY` all
      # survive, so these tags keep their primary keys where the sqlite ones
      # cannot. Row counts match the sqlite tags exactly. Each dataset's
      # scripts/<dataset> dir is a symlink to scripts/pgsql.
      world:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/world/world-1.0/world-1.0.tar.gz
        sqlFiles: dbsamples-0.1/world/world.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/world.yaml
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/iso3166.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/frenchtowns.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/usda.yaml
      pgexercises:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/pgexercises.yaml
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/dellstore.yaml
      # Northwind specialty-foods sample, same Yugabyte PostgreSQL-dialect dump
      # behind the postgres `northwind` and cockroach `northwind` tags
      # (DDL + multi-row INSERT data). Everything but one token goes through the
      # shared scripts/pgsql hook unchanged; the exception is the empty bytea
      # literal `'\x'` the dump writes for categories.picture and
      # employees.photo, which DuckDB's BLOB parser rejects. scripts/northwind
      # is therefore a thin transform hook that turns on the shared hook's
      # `'\x'` -> `''` knob and delegates. Row counts match the cockroach tag
      # exactly.
      northwind:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/northwind_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/northwind_data.sql
        sqlFiles: northwind_ddl.sql northwind_data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/northwind.yaml
      # SportsDB, the same five-file Yugabyte dump the cockroach/mysql/sqlite
      # sportsdb tags use: 107 tables of cross-sport statistics, loaded from
      # single-row INSERTs. Handled by the shared scripts/pgsql hook (a plain
      # symlink) -- its `CREATE DOMAIN` drop, `serial` demotion, `USING lsm`
      # strip and constraint filtering are all this dump needs. Row counts match
      # the sqlite tag exactly.
      sportsdb:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_tables.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_inserts.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_indexes.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_constraints.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/sportsdb_fks.sql
        sqlFiles: sportsdb_tables.sql sportsdb_inserts.sql sportsdb_indexes.sql sportsdb_constraints.sql sportsdb_fks.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/sportsdb.yaml
      # MoMA research collection. Published only as CSV, so the schema is
      # authored in-repo (duckdb/scripts/moma) and the CSVs are bulk-loaded at
      # build time by the `COPY ... FROM '<file>.csv'` statements the transform
      # hook appends -- DuckDB reads CSV in plain SQL, so unlike the sqlite tag
      # there are no dot-commands involved. Counts drift between builds (the
      # CSVs are refreshed upstream), so the smoke test records floors.
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        sqlFiles: moma.sql
        structureTest:
          configs:
            - test/config/duckdb/moma.yaml
      # GeoNames cities15000. Published only as a zipped tab-separated export,
      # so the schema is authored in-repo (duckdb/scripts/geonames), the extract
      # hook unzips it and the transform hook appends the one `COPY cities FROM
      # '<file>.txt'` that bulk-loads it at build time -- with quote processing
      # switched off, since a double quote in a place name is a literal
      # character. Upstream refreshes the dump daily, so the smoke test records
      # floors.
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        sqlFiles: geonames.sql
        structureTest:
          configs:
            - test/config/duckdb/geonames.yaml
      # OpenFlights airports/airlines/routes. Published only as three
      # comma-separated data files, so the schema is authored in-repo
      # (duckdb/scripts/openflights) and the files are bulk-loaded at build time
      # by the three `COPY <table> FROM '<file>.dat'` statements the transform
      # hook appends -- already RFC4180 CSV, so they need no normalizing, only
      # the upstream's `\N` mapped to NULL. Upstream refreshes them in place, so
      # the smoke test records floors.
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        sqlFiles: openflights.sql
        structureTest:
          configs:
            - test/config/duckdb/openflights.yaml
      # Airlines: the postgrespro flight-bookings demo (the same pinned
      # snapshot the postgres and sqlite tags use), a single gzipped pg_dump
      # that manages its own `demo` database and qualifies everything to a
      # `bookings` schema. At 10.7M rows it is by far the biggest dataset here,
      # so the scripts/airlines transform turns on the shared hook's
      # PGSQL_COPY_CSV knob: each COPY block is written out as a sibling CSV
      # and loaded by DuckDB's vectorized reader rather than parsed as INSERT
      # text, which is what keeps the LOAD step at seconds rather than hours.
      # The hook also enables the schema-flattening, view-dropping and
      # type-mapping knobs (`jsonb` -> `json`, and `point` / `tstzrange` /
      # `integer[]` -> `text`, since DuckDB has none of those types) and
      # removes what no knob covers -- three extensions, the ALTER DATABASE
      # settings, the ANY(ARRAY[...]) spelling of three CHECK constraints, and
      # the index over a range bound. All nine tables keep their comments and
      # their primary keys; row counts match the postgres and sqlite tags
      # exactly. See the hook's docstring for the DuckDB error each rewrite
      # avoids.
      airlines:
        extractUrl: https://edu.postgrespro.ru/demo-20250901-3m.sql.gz
        sqlFiles: airlines.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/airlines.yaml
      # NYC TLC yellow-taxi trip records -- the DuckDB flagship, and the one
      # dataset here that ships as Parquet rather than SQL or CSV. Parquet is
      # compiled into the duckdb CLI, so the scripts/nyc-taxi transform hook
      # only has to author the one `CREATE TABLE trips AS FROM read_parquet(...)`
      # statement the generic LOAD step runs; the 73.5 MB file itself stays
      # behind in the loader stage (only /data is copied into the image).
      #
      # The month is pinned rather than tracking "latest available" so the tag
      # is reproducible: TLC publishes a new file every month and revises older
      # ones, and an unpinned URL would make the row count -- and so the smoke
      # test's expectation -- drift under CI. 2025-06 is ~4M rows.
      nyc-taxi:
        extractUrl: https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-06.parquet
        sqlFiles: nyc-taxi.sql
        structureTest:
          configs:
            - test/config/duckdb/nyc-taxi.yaml
      # Sakila, the MySQL DVD-rental sample (the original of pagila), from
      # jOOQ's multi-dialect port collection -- here the PostgreSQL flavour,
      # schema plus the COPY-format data dump. The repo is cloned (like the
      # mysql `northwind` tag) but the data is fixed, so counts are recorded
      # exactly, and they match the cockroach `sakila` tag table for table.
      #
      # This is the richest PostgreSQL dump in the set, so unlike the six tags
      # above it does not symlink straight to scripts/pgsql: the
      # scripts/sakila hook (python3) first removes or maps what that shared
      # hook does not cover -- a CREATE DOMAIN, a user-defined aggregate, eight
      # functions (three with a terminator the shared hook's regex misses),
      # fourteen triggers, six INHERITS partition children with their rules and
      # indexes, `tsvector` and `text[]` columns, a `USING gist` index and the
      # `CACHE` clause on every sequence -- and then execs it. All seven views
      # survive; the tag ships the same 15 base tables as the cockroach one.
      # See the hook's docstring for the DuckDB error each rewrite avoids.
      sakila:
        extractUrl: https://github.com/jOOQ/sakila.git
        sqlFiles: sakila/postgres-sakila-db/postgres-sakila-schema.sql sakila/postgres-sakila-db/postgres-sakila-insert-data-using-copy.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/duckdb/sakila.yaml
      # StackExchange sites. Each ships as per-table XML in a 7z archive (no
      # SQL), so -- exactly like the other engines -- every site builds through
      # the shared scripts/stackexchange hooks (each site's scripts/<site> dir
      # is a symlink to it): the extract hook un-7z's the dump and the transform
      # hook converts the XML to a DuckDB init script (CREATE TABLE + batched
      # INSERTs + indexes). Adding a site is just a tag here + a structure-test
      # config + an expected/duckdb/<site>.json floor file.
      stackexchange-beer:
        dataset: beer
        extractUrl: https://archive.org/download/stackexchange/beer.stackexchange.com.7z
        sqlFiles: beer.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-beer.yaml
      stackexchange-coffee:
        dataset: coffee
        extractUrl: https://archive.org/download/stackexchange/coffee.stackexchange.com.7z
        sqlFiles: coffee.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-coffee.yaml
      stackexchange-poker:
        dataset: poker
        extractUrl: https://archive.org/download/stackexchange/poker.stackexchange.com.7z
        sqlFiles: poker.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-poker.yaml
      stackexchange-woodworking:
        dataset: woodworking
        extractUrl: https://archive.org/download/stackexchange/woodworking.stackexchange.com.7z
        sqlFiles: woodworking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-woodworking.yaml
      stackexchange-chess:
        dataset: chess
        extractUrl: https://archive.org/download/stackexchange/chess.stackexchange.com.7z
        sqlFiles: chess.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-chess.yaml
      stackexchange-cooking:
        dataset: cooking
        extractUrl: https://archive.org/download/stackexchange/cooking.stackexchange.com.7z
        sqlFiles: cooking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-cooking.yaml
      stackexchange-outdoors:
        dataset: outdoors
        extractUrl: https://archive.org/download/stackexchange/outdoors.stackexchange.com.7z
        sqlFiles: outdoors.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-outdoors.yaml
      stackexchange-boardgames:
        dataset: boardgames
        extractUrl: https://archive.org/download/stackexchange/boardgames.stackexchange.com.7z
        sqlFiles: boardgames.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/duckdb/stackexchange-boardgames.yaml
      # `latest` tracks a single representative dataset, mirroring the other
      # engines.
      latest:
        retagFrom: chinook
        structureTest: false
  # Apache Pinot images, published under their own repository. Pinot is a
  # distributed OLAP store, not a SQL database with a bulk-load path: its unit
  # of storage is an immutable segment, its DDL is a pair of JSON documents per
  # table (a schema and a table config) POSTed to a running controller, and the
  # official image's entrypoint is `pinot-admin.sh` rather than anything that
  # replays init scripts. So the build cannot hand a dump to a client the way
  # postgres/cockroach do, and cannot bake a database file the way sqlite/duckdb
  # do. Instead each tag's transform hook decompiles its source into per-table
  # CSV + schema + table config + a manifest of exact row counts, and the
  # image's wrapper entrypoint starts a one-container cluster (ZooKeeper, then
  # controller + broker + server) and replays them over REST at container start
  # -- still the repo's init-at-start model, carried over HTTP instead of SQL.
  # The build templates are the Pinot flavours (no init-script assembly; test
  # runs the Pinot smoke-test script) and override the global postgres ones for
  # this context.
  pinot:
    tagKeys: dataset
    parameters:
      repository: aa8y/pinot-dataset
    structureTest:
      configs:
        - test/config/pinot/common.yaml
    templates:
      # Mirrors the global templates (see top of file), including the always-on
      # registry build cache gated by the DATASET_CHECKSUM fingerprint.
      # SQL_FILES is emitted per tag (a Mustache section) because the datasets
      # sourced from published CSV/TSV have no SQL input to name.
      build: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-from type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
        ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
        ${CACHE_TO_SCOPE:+--cache-to type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        {{#sqlFiles}}--build-arg 'SQL_FILES={{{.}}}'{{/sqlFiles}}
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
      test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run-pinot.sh {{tag}} {{dataset}}{{/retagFrom}}'
      push: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-to type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        {{#sqlFiles}}--build-arg 'SQL_FILES={{{.}}}'{{/sqlFiles}}
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
    tags:
      # PostgreSQL-dump datasets, the same sources the sqlite/duckdb tags use.
      # None of them can be *run* on Pinot, so the shared scripts/pgsql hook
      # decompiles each dump instead: it parses the CREATE TABLE DDL for column
      # names and types, reads the COPY blocks (world, iso3166, frenchtowns,
      # usda, dellstore) and multi-row INSERTs (pgexercises) for the data, and
      # emits one CSV plus a Pinot schema and OFFLINE table config per table,
      # with the exact row count recorded in tables.tsv so the entrypoint can
      # wait for it. Row counts match the postgres/duckdb tags exactly. Each
      # dataset's scripts/<dataset> dir is a symlink to scripts/pgsql.
      world:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/world/world-1.0/world-1.0.tar.gz
        sqlFiles: dbsamples-0.1/world/world.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/world.yaml
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/iso3166.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/frenchtowns.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/usda.yaml
      pgexercises:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/pgexercises.yaml
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/dellstore.yaml
      # Datasets published only as CSV/TSV, with no SQL anywhere upstream. Their
      # schemas are authored in-repo (pinot/scripts/<dataset>/schema.sql, in
      # plain PostgreSQL DDL exactly as the cockroach tags do) and their
      # transform hooks parse that DDL with the *same* CREATE TABLE reader the
      # scripts/pgsql hook uses on real dumps -- so one PostgreSQL-type-to-Pinot
      # mapping serves every tag in this context. All three are refreshed
      # upstream in place, so their smoke tests record floors, not exact counts.
      #
      # GeoNames cities15000: a zipped tab-separated export with no header and
      # no quoting at all, so the extract hook unzips it and the transform
      # splits on tabs rather than parsing CSV.
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/geonames.yaml
      # OpenFlights airports/airlines/routes: three RFC4180 CSV files with no
      # header, using an unquoted \N for a missing value -- which is already the
      # NULL sentinel the entrypoint hands Pinot.
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/openflights.yaml
      # MoMA research collection: ~177k rows, and the bulkiest tag here (82 MB
      # of shipped CSV, since artworks is 30 columns of free text; usda has
      # more rows in a fifth of the space).
      # Every column is STRING, deliberately -- the exports carry approximate
      # dates and blank measurements, and on Pinot a value a column's type
      # cannot parse becomes that type's *default* rather than an error.
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/pinot/moma.yaml
      # `latest` tracks a single representative dataset, mirroring the other
      # engines. world rather than chinook: chinook is not ported here, and
      # world is the tag this engine's README uses in every example.
      latest:
        retagFrom: world
        structureTest: false
  # ClickHouse images. Built on the official clickhouse/clickhouse-server
  # Alpine image and published under a separate repository. ClickHouse is the
  # first columnar OLAP engine here, so its *data model* differs from the
  # PostgreSQL sample dumps and not just its SQL spelling: every table names an
  # engine (MergeTree) and a sorting key, columns are NOT NULL unless declared
  # Nullable, and there are no constraints, sequences or secondary indexes. The
  # dumps therefore go through a real dialect translation
  # (clickhouse/scripts/pgsql/transform) rather than the thin filters the
  # DuckDB and CockroachDB contexts use. The official entrypoint honours the
  # same /docker-entrypoint-initdb.d convention the postgres image does (plus a
  # CLICKHOUSE_DB env var), so -- like the other server engines -- these images
  # ship an init script and load on first container start. The build templates
  # are the ClickHouse flavours (same arg surface as cockroach minus the
  # index/cd knobs; test runs the ClickHouse smoke-test script) and override
  # the global postgres ones for this context.
  clickhouse:
    tagKeys: dataset
    parameters:
      repository: aa8y/clickhouse-dataset
    structureTest:
      configs:
        - test/config/clickhouse/common.yaml
    templates:
      # Mirrors the global templates (see top of file), including the always-on
      # registry build cache gated by the DATASET_CHECKSUM fingerprint.
      build: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-from type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
        ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
        ${CACHE_TO_SCOPE:+--cache-to type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
      test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run-clickhouse.sh {{tag}} {{dataset}}{{/retagFrom}}'
      push: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-to type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
    tags:
      # PostgreSQL-dump datasets -- the pgFoundry family plus the two Yugabyte
      # mirrors -- all from the same sources the duckdb/cockroach tags use and
      # all through the shared scripts/pgsql transform hook (each dataset's
      # scripts/<dataset> dir is a symlink to scripts/pgsql). That hook rebuilds
      # every CREATE TABLE with ClickHouse types, a MergeTree engine and the
      # dump's primary key as the sorting key, turns COPY blocks (and chinook's
      # 15,607 single-row INSERTs) into batched multi-row INSERTs, and doubles
      # the backslashes ClickHouse would otherwise read as escapes. Row counts
      # match the postgres/duckdb tags exactly.
      world:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/world/world-1.0/world-1.0.tar.gz
        sqlFiles: dbsamples-0.1/world/world.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/clickhouse/world.yaml
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/clickhouse/iso3166.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/clickhouse/frenchtowns.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/clickhouse/usda.yaml
      pgexercises:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/clickhouse/pgexercises.yaml
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/clickhouse/dellstore.yaml
      chinook:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_genres_artists_albums.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/chinook_songs.sql
        sqlFiles: chinook_ddl.sql chinook_genres_artists_albums.sql chinook_songs.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/clickhouse/chinook.yaml
      # NYC TLC yellow-taxi trip records -- the ClickHouse flagship here, and
      # the only dataset that ships as Parquet. The scripts/nyc_taxi transform
      # hook authors an explicit CREATE TABLE (the TLC's own column names, types
      # read out of the Parquet schema) plus the `INSERT ... FROM INFILE ...
      # FORMAT Parquet` that loads it; the 73.5 MB file ships in the image,
      # because unlike DuckDB this engine loads at container start.
      #
      # The month is pinned rather than tracking "latest available" so the tag
      # is reproducible: TLC publishes a new file every month and revises older
      # ones, and an unpinned URL would make the row count -- and so the smoke
      # test's expectation -- drift under CI. 2025-06 is 4,322,960 rows.
      #
      # `dataset: nyc_taxi` (with an underscore) while the tag keeps the hyphen
      # every other engine uses: the database name has to be a bare ClickHouse
      # identifier, since the official entrypoint interpolates CLICKHOUSE_DB
      # straight into `CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB` unquoted,
      # and `nyc-taxi` there is a subtraction.
      nyc-taxi:
        dataset: nyc_taxi
        extractUrl: https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-06.parquet
        sqlFiles: nyc_taxi.sql
        structureTest:
          configs:
            - test/config/clickhouse/nyc-taxi.yaml
      # Datasets published only as CSV/TSV, so their schemas are authored
      # in-repo (clickhouse/scripts/<dataset>/schema.sql, same columns and types
      # as the other engines') and the transform hook appends an
      # `INSERT ... FROM INFILE` per table. The data files ship next to the init
      # script and are read by the client at container start. All three are
      # refreshed in place upstream, so the smoke test records floors.
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        sqlFiles: moma.sql
        structureTest:
          configs:
            - test/config/clickhouse/moma.yaml
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        sqlFiles: geonames.sql
        structureTest:
          configs:
            - test/config/clickhouse/geonames.yaml
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        sqlFiles: openflights.sql
        structureTest:
          configs:
            - test/config/clickhouse/openflights.yaml
      # StackExchange sites. Each ships as per-table XML in a 7z archive (no
      # SQL), so -- exactly like the other engines -- every site builds through
      # the shared scripts/stackexchange hooks (each site's scripts/<site> dir
      # is a symlink to it): the extract hook un-7z's the dump and the transform
      # hook converts the XML to a ClickHouse init script (MergeTree tables
      # keyed on Id + batched INSERTs; no indexes, which ClickHouse has no
      # equivalent of). Adding a site is just a tag here + a structure-test
      # config + an expected/clickhouse/<site>.json floor file.
      stackexchange-beer:
        dataset: beer
        extractUrl: https://archive.org/download/stackexchange/beer.stackexchange.com.7z
        sqlFiles: beer.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-beer.yaml
      stackexchange-coffee:
        dataset: coffee
        extractUrl: https://archive.org/download/stackexchange/coffee.stackexchange.com.7z
        sqlFiles: coffee.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-coffee.yaml
      stackexchange-poker:
        dataset: poker
        extractUrl: https://archive.org/download/stackexchange/poker.stackexchange.com.7z
        sqlFiles: poker.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-poker.yaml
      stackexchange-woodworking:
        dataset: woodworking
        extractUrl: https://archive.org/download/stackexchange/woodworking.stackexchange.com.7z
        sqlFiles: woodworking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-woodworking.yaml
      stackexchange-chess:
        dataset: chess
        extractUrl: https://archive.org/download/stackexchange/chess.stackexchange.com.7z
        sqlFiles: chess.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-chess.yaml
      stackexchange-cooking:
        dataset: cooking
        extractUrl: https://archive.org/download/stackexchange/cooking.stackexchange.com.7z
        sqlFiles: cooking.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-cooking.yaml
      stackexchange-outdoors:
        dataset: outdoors
        extractUrl: https://archive.org/download/stackexchange/outdoors.stackexchange.com.7z
        sqlFiles: outdoors.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-outdoors.yaml
      stackexchange-boardgames:
        dataset: boardgames
        extractUrl: https://archive.org/download/stackexchange/boardgames.stackexchange.com.7z
        sqlFiles: boardgames.sql
        extraPrereqs: p7zip python3 py3-lxml
        structureTest:
          configs:
            - test/config/clickhouse/stackexchange-boardgames.yaml
      # `latest` tracks a single representative dataset, mirroring the other
      # engines. nyc-taxi rather than chinook here: it is the one tag that shows
      # what this engine is for -- 4.3M rows of Parquet in a MergeTree table.
      latest:
        retagFrom: nyc-taxi
        structureTest: false
  # Apache Druid images. Built on the multi-arch Eclipse Temurin JRE from the
  # pinned Apache Druid binary release rather than FROM apache/druid, which is
  # published for amd64 only (its own build compiles the web console, which does
  # not resolve on arm64) while this repo ships both architectures -- see
  # druid/Dockerfile. Druid is a real-time OLAP store with no DDL +
  # INSERT/COPY load path: data enters only through an ingestion task that
  # writes immutable segments, so the build cannot bake a loaded database into
  # the image the way the DuckDB/SQLite builds do. Instead every dataset ships
  # as one CSV plus one native batch ingestion spec per table, and the
  # entrypoint submits them on first boot and waits for the segments to load
  # before announcing readiness. One table becomes one datasource of the same
  # name, so the smoke test's keys read like the other engines'.
  druid:
    tagKeys: dataset
    parameters:
      repository: aa8y/druid-dataset
    structureTest:
      configs:
        - test/config/druid/common.yaml
    templates:
      # Mirrors the global templates (see top of file), including the always-on
      # registry build cache gated by the DATASET_CHECKSUM fingerprint.
      build: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-from type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}
        ${CACHE_FROM_SCOPE:+--cache-from type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_FROM_SCOPE}}
        ${CACHE_TO_SCOPE:+--cache-to type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-}-${CACHE_TO_SCOPE},mode=max,image-manifest=true}
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}}{{/retagFrom}}
      test: '{{^retagFrom}}REPOSITORY={{{repository}}} bash test/integration/with-retry.sh run-druid.sh {{tag}} {{dataset}}{{/retagFrom}}'
      push: >
        {{^retagFrom}}docker buildx build --load ${PLATFORM:+--platform "$PLATFORM"}
        --cache-to type=registry,ref={{{repository}}}:buildcache-{{tag}}${CACHE_ARCH:-},mode=max,image-manifest=true
        --build-arg DATASET={{dataset}}
        --build-arg DATASET_CHECKSUM="$(bin/dataset-checksum '{{{extractUrl}}}')"
        --build-arg 'EXTRACT_URL={{{extractUrl}}}'
        --build-arg 'SQL_FILES={{{sqlFiles}}}'
        {{#extraPrereqs}}--build-arg 'EXTRA_PREREQS={{{.}}}'{{/extraPrereqs}}
        -t {{{repository}}}:{{tag}} {{{context}}} && bin/push-image {{{repository}}}:{{tag}}{{/retagFrom}}
    tags:
      # PostgreSQL-dump datasets, the same sources the DuckDB and SQLite tags
      # use. None of the dumps' SQL survives -- Druid runs none of it -- so the
      # shared scripts/pgsql hook reads them instead of rewriting them: it
      # parses the CREATE TABLE DDL for column names, types and NOT NULL flags,
      # decodes the COPY / multi-row INSERT data, and writes one CSV plus one
      # ingestion spec per table. Row counts match the DuckDB tags exactly
      # (dellstore aside; see its comment). Each dataset's scripts/<dataset> dir
      # is a symlink to scripts/pgsql. SQL_FILES keeps its name for manifest
      # uniformity: here it names the ordered list of source files the transform
      # reads, not files fed to a SQL client.
      world:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/world/world-1.0/world-1.0.tar.gz
        sqlFiles: dbsamples-0.1/world/world.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/world.yaml
      iso3166:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/iso-3166/iso-3166-1.0/iso-3166-1.0.tar.gz
        sqlFiles: iso-3166/iso-3166.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/iso3166.yaml
      frenchtowns:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/french-towns-communes-francais/french-towns-communes-francaises-1.0/french-towns-communes-francaises-1.0.tar.gz
        sqlFiles: french-towns-communes-francaises.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/frenchtowns.yaml
      usda:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/usda/usda-r18-1.0/usda-r18-1.0.tar.gz
        sqlFiles: usda-r18-1.0/usda.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/usda.yaml
      pgexercises:
        extractUrl: >-
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_ddl.sql
          https://raw.githubusercontent.com/yugabyte/yugabyte-db/master/sample/clubdata_data.sql
        sqlFiles: clubdata_ddl.sql clubdata_data.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/pgexercises.yaml
      # dellstore ships one table, `reorder`, with zero rows. Druid has no empty
      # datasource -- an ingestion task that produces no segments succeeds and
      # leaves nothing behind to query -- so the transform skips it (loudly, at
      # build time) and this tag carries seven datasources where the other
      # engines have eight. Every other count matches them exactly.
      dellstore:
        extractUrl: https://ftp.postgresql.org/pub/projects/pgFoundry/dbsamples/dellstore2/dellstore2-normal-1.0/dellstore2-normal-1.0.tar.gz
        sqlFiles: dellstore2-normal-1.0/dellstore2-normal-1.0.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/dellstore.yaml
      # Datasets published only as delimited text, whose schema is therefore
      # authored in-repo (druid/scripts/<dataset>/schema.sql, the same schemas
      # the CockroachDB tags use). Their transform hooks do not reimplement the
      # Druid half: they assemble the PostgreSQL dump the upstream never shipped
      # -- the authored DDL plus a `COPY ... FROM stdin` block per file -- and
      # hand it to the shared scripts/pgsql hook, which already turns DDL + COPY
      # into typed ingestion specs and is unit-tested for it. All three fetch
      # from a live upstream at build time, so the smoke test records floors.
      #
      # geonames is the cheapest of the three by a distance: its published
      # export *is* COPY TEXT (tab-separated, unquoted, unescaped), so its hook
      # is ten lines of shell and nothing is re-encoded. It is also the one
      # dataset here whose `__time` is real data -- every row carries a
      # modification date.
      geonames:
        extractUrl: https://download.geonames.org/export/dump/cities15000.zip
        sqlFiles: geonames.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/geonames.yaml
      openflights:
        extractUrl: >-
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/airlines.dat
          https://raw.githubusercontent.com/jpatokal/openflights/master/data/routes.dat
        sqlFiles: openflights.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/openflights.yaml
      # MoMA's exports are the messiest data in this repo -- ragged rows,
      # embedded newlines and quotes. Rows are padded/truncated to the schema's
      # width on the way into the dump, and the shared hook flattens embedded
      # newlines when it writes the Druid CSV, because Druid's CSV reader splits
      # input into lines before parsing them.
      moma:
        extractUrl: >-
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artworks.csv
          https://media.githubusercontent.com/media/MuseumofModernArt/collection/main/Artists.csv
        sqlFiles: moma.sql
        extraPrereqs: python3
        structureTest:
          configs:
            - test/config/druid/moma.yaml
      # `latest` tracks a single representative dataset, mirroring the other
      # engines. world rather than chinook: chinook is not ported here (its
      # upstreams are a prebuilt SQLite database and a PostgreSQL-dialect dump
      # of quoted CamelCase DDL + INSERTs), and world is this context's flagship.
      latest:
        retagFrom: world
        structureTest: false