From 4b687dcfbef7a88e858be91913ab60158a180a44 Mon Sep 17 00:00:00 2001
From: George Robertson <41301972+georgeRobertson2@users.noreply.github.com>
Date: Wed, 27 May 2026 17:27:55 +0100
Subject: [PATCH 01/17] refactor: upgrade pydantic to v2 (#107)
* build: uplift pydantic to v2
* style: formatting, linting fixes and additional testing on audit record
---
poetry.lock | 405 ++++++++++++++----
pyproject.toml | 5 +-
src/dve/core_engine/backends/base/auditing.py | 16 +-
.../implementations/duckdb/auditing.py | 16 +-
.../implementations/duckdb/contract.py | 17 +-
.../implementations/duckdb/duckdb_helpers.py | 14 +-
.../implementations/duckdb/readers/csv.py | 8 +-
.../implementations/duckdb/readers/json.py | 4 +-
.../implementations/duckdb/readers/xml.py | 4 +-
.../implementations/spark/auditing.py | 2 +-
.../backends/implementations/spark/backend.py | 2 +-
.../implementations/spark/spark_helpers.py | 49 ++-
.../implementations/spark/utilities.py | 2 +-
.../core_engine/backends/metadata/contract.py | 6 +-
.../backends/metadata/reporting.py | 16 +-
.../core_engine/backends/metadata/rules.py | 52 +--
src/dve/core_engine/backends/readers/csv.py | 6 +-
.../core_engine/backends/readers/utilities.py | 2 +-
src/dve/core_engine/backends/readers/xml.py | 30 +-
src/dve/core_engine/backends/utilities.py | 42 +-
.../core_engine/configuration/v1/__init__.py | 12 +-
src/dve/core_engine/configuration/v1/steps.py | 17 +-
src/dve/core_engine/engine.py | 81 ++--
src/dve/core_engine/loggers.py | 2 +-
src/dve/core_engine/message.py | 27 +-
src/dve/core_engine/models.py | 50 ++-
src/dve/core_engine/validation.py | 9 +-
src/dve/metadata_parser/domain_types.py | 158 ++++---
src/dve/metadata_parser/function_library.py | 4 +-
src/dve/metadata_parser/function_wrapper.py | 41 +-
src/dve/metadata_parser/model_generator.py | 16 +-
src/dve/metadata_parser/models.py | 94 ++--
.../file_handling/implementations/file.py | 8 +-
src/dve/pipeline/pipeline.py | 6 +-
src/dve/pipeline/utils.py | 6 +-
tests/features/steps/steps_post_pipeline.py | 2 +-
tests/fixtures.py | 3 -
.../test_duckdb/test_audit_ddb.py | 4 +-
.../test_duckdb/test_data_contract.py | 12 +-
.../test_duckdb/test_duckdb_helpers.py | 4 +-
.../test_duckdb/test_rules.py | 4 +-
.../test_spark/test_audit_spark.py | 4 +-
.../test_spark/test_rules.py | 4 +-
.../test_spark/test_spark_helpers.py | 4 +-
.../test_backends/test_readers/test_csv.py | 4 +-
.../test_readers/test_ddb_csv.py | 4 +-
.../test_readers/test_ddb_json.py | 8 +-
.../test_readers/test_spark_json.py | 8 +-
.../test_backends/test_utilities.py | 55 +++
tests/test_core_engine/test_message.py | 13 +-
tests/test_core_engine/test_models.py | 25 +-
.../test_domain_types.py | 18 +-
tests/test_pipeline/test_spark_pipeline.py | 8 +-
53 files changed, 929 insertions(+), 484 deletions(-)
create mode 100644 tests/test_core_engine/test_backends/test_utilities.py
diff --git a/poetry.lock b/poetry.lock
index dfe26c4..ec61a6d 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,5 +1,17 @@
# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand.
+[[package]]
+name = "annotated-types"
+version = "0.7.0"
+description = "Reusable constraint types to use with typing.Annotated"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
+ {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
+]
+
[[package]]
name = "argcomplete"
version = "3.6.3"
@@ -1376,6 +1388,106 @@ files = [
{file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"},
]
+[[package]]
+name = "librt"
+version = "0.11.0"
+description = "Mypyc runtime library"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev", "lint"]
+files = [
+ {file = "librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f"},
+ {file = "librt-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ae627397a2f351560440d872d6f7c8dbb4072e57868e7b2fc5b8b430fe489d45"},
+ {file = "librt-0.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc329359321b67d24efdf4bc69012b0597001649544db662c001db5a0184794c"},
+ {file = "librt-0.11.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:7e82e642ab0f7608ce2fe53d76ca2280a9ee33a1b06556142c7c6fe80a86fc33"},
+ {file = "librt-0.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88145c15c67731d54283d135b03244028c750cc9edc334a96a4f5950ebdb2884"},
+ {file = "librt-0.11.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d36a51b3d93320b686588e27123f4995804dbf1bce81df78c02fc3c6eea9280"},
+ {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3ac06a2a8b246327f11e186a53a100a4d5c7ed52346367e5ec751d51586c"},
+ {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:461bbceede621f1ffb8839755f8663e886087ee7af16294cab7fb4d782c62eeb"},
+ {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0cad8a4d6a8ff03c9b76f9414caccd78e7cfbc8a2e12fa334d8e1d9932753783"},
+ {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f37aa505b3cf60701562eddb32df74b12a9e380c207fd8b06dd157a943ac7ea0"},
+ {file = "librt-0.11.0-cp310-cp310-win32.whl", hash = "sha256:94663a21534637f0e787ec2a2a756022df6e5b7b2335a5cdd7d8e33d68a2af89"},
+ {file = "librt-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:dec7db73758c2b54953fd8b7fe348c45188fe26b39ee18446196edd08453a5d4"},
+ {file = "librt-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29"},
+ {file = "librt-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9"},
+ {file = "librt-0.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5"},
+ {file = "librt-0.11.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5d63c855d86938d9de93e265c9bd8c705b51ec494de5738340ee93767a686e4b"},
+ {file = "librt-0.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993f028be9e96a08d31df3479ac80d99be374d17f3b78e4796b3fd3c913d4e89"},
+ {file = "librt-0.11.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:258d73a0aa66a055e65b2e4d1b8cdb23b9d132c5bb915d9547d804fcaed116cc"},
+ {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0827efe7854718f04aaddf6496e96960a956e676fe1d0f04eb41511fd8ad06d5"},
+ {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7753e57d6e12d019c0d8786f1c09c709f4c3fcc57c3887b24e36e6c06ec938b7"},
+ {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:11bd19822431cc21af9f27374e7ae2e58103c7d98bda823536a6c47f6bb2bb3d"},
+ {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:22bdf239b219d3993761a148ffa134b19e52e9989c84f845d5d7b71d70a17412"},
+ {file = "librt-0.11.0-cp311-cp311-win32.whl", hash = "sha256:46c60b61e308eb535fbd6fa622b1ee1bb2815691c1ad9c98bf7b84952ec3bc8d"},
+ {file = "librt-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:902e546ff044f579ff1c953ff5fce97b636fe9e3943996b2177710c6ef076f73"},
+ {file = "librt-0.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:65ac3bc20f78aa0ee5ae84baa68917f89fef4af63e941084dd019a0d0e749f0c"},
+ {file = "librt-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b87504f1690a23b9a2cca841191a04f83895d4fc2dd04df91d82b1a04ca2ad46"},
+ {file = "librt-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40071fc5fe0ce8daa6de616702314a01e1250711682b0523d6ab8d4525910cb3"},
+ {file = "librt-0.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:137e79445c896a0ea7b265f52d23954e05b64222ee1af69e2cb34219067cbb67"},
+ {file = "librt-0.11.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:cca6644054e78746d8d4ef238681f9c34ff8b584fe6b988ecebb8db3b15e622a"},
+ {file = "librt-0.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5b0eea49f5562861ee8d757a32ef7d559c1d35be2aaaa1ec28941d74c9ffc8a"},
+ {file = "librt-0.11.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0d1029d7e1ae1a7e647ed6fb5df8c4ce2dffefb7a9f5fd1376a4554d96dac09f"},
+ {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bc3ce6b33c5828d9e80592011a5c584cb2ce86edbc4088405f70da47dc1d1b3b"},
+ {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:936c5995f3514a42111f20099397d8177c79b4d7e70961e396c6f5a0a3566766"},
+ {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9bc0ca6ad9381cbe8e4aa6e5726e4c80c78115a6e9723c599ed1d73e092bc49d"},
+ {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:070aa8c26c0a74774317a72df8851facc7f0f012a5b406557ac56992d92e1ec8"},
+ {file = "librt-0.11.0-cp312-cp312-win32.whl", hash = "sha256:6bf14feb84b05ae945277395451998c89c54d0def4070eb5c08de544930b245a"},
+ {file = "librt-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:75672f0bc524ede266287d532d7923dbce94c7514ad07627bac3d0c6d92cc4d9"},
+ {file = "librt-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c"},
+ {file = "librt-0.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:78dc31f7fdfe9c9d0eb0e8f42d139db230e826415bbcabd9f0e9faaaee909894"},
+ {file = "librt-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fa475675db22290c3158e1d42326d0f5a65f04f44a0e68c3630a25b53560fb9c"},
+ {file = "librt-0.11.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:621db29691044bdeda22e789e482e1b0f3a985d90e3426c9c6d17606416205ea"},
+ {file = "librt-0.11.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:a9010e2ed5b3a9e158c5fd966b3ab7e834bb3d3aacc8f66c91dd4b57a3799230"},
+ {file = "librt-0.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c39513d8b7477a2e1ed8c43fc21c524e8d5a0f8d4e8b7b074dbdbe7820a08e2"},
+ {file = "librt-0.11.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7aef3cf1d5af86e770ab04bfd993dfc4ae8b8c17f66fb77dd4a7d50de7bbb1a3"},
+ {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:557183ddc36babe46b27dd60facbd5adb4492181a5be887587d57cda6e092f21"},
+ {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:83d3e1f72bd42f6c5c0b7daec530c3f829bd02db42c70b8ddf0c2d90a2459930"},
+ {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:4ce1f21fbe589bc1afd7872dece84fb0e1144f794a288e58a10d2c54a55c43be"},
+ {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b09f7044ea2b64c9da42fd3d335666518cfd1c6e8a182c95da73d0214b41e"},
+ {file = "librt-0.11.0-cp313-cp313-win32.whl", hash = "sha256:78fddc31cd4d3caa897ad5d31f856b1faadc9474021ad6cb182b9018793e254e"},
+ {file = "librt-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ca8aa88751a775870b764e93bad5135385f563cb8dcee399abf034ea4d3cb47"},
+ {file = "librt-0.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:96f044bb325fd9cf1a723015638c219e9143f0dfbc0ca54c565df2b7fc748b44"},
+ {file = "librt-0.11.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4a017a95e5837dc15a8c5661d60e05daa96b90908b1aa6b7acdf443cd25c8ebd"},
+ {file = "librt-0.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b1ecbd9819deccc39b7542bf4d2a740d8a620694d39989e58661d3763458f8d4"},
+ {file = "librt-0.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7da327dacd7be8f8ec36547373550744a3cc0e536d54665cd83f8bcd961200e8"},
+ {file = "librt-0.11.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0dc56b1f8d06e60db362cc3fdae206681817f86ce4725d34511473487f12a34b"},
+ {file = "librt-0.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05fb8fb2ab90e21c8d12ea240d744ad514da9baf381ebfa70d91d20d21713175"},
+ {file = "librt-0.11.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae74872be221df4374d10fec61f93ed1513b9546ea84f2c0bf73ab3e9bd0b03"},
+ {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32bcc918c0148eb7e3d57385125bac7e5f9e4359d05f07448b09f6f778c2f31c"},
+ {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f9743fc99135d5f78d2454435615f6dec0473ca507c26ce9d92b10b562a280d3"},
+ {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5ba067f4aadae8fda802d91d2124c90c42195ff32d9161d3549e6d05cfe26f96"},
+ {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:de3bf945454d032f9e390b85c4072e0a0570bf825421c8be0e71209fa65e1abe"},
+ {file = "librt-0.11.0-cp314-cp314-win32.whl", hash = "sha256:d2277a05f6dcb9fd13db9566aac4fabd68c3ea1ea46ee5567d4eef8efa495a2f"},
+ {file = "librt-0.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ab73e8db5e3f564d812c1f5c3a175930a5f9bc96ccb5e3b22a34d7858b401cf7"},
+ {file = "librt-0.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:aea3caa317752e3a466fa8af45d91ee0ea8c7fdd96e42b0a8dd9b76a7931eba1"},
+ {file = "librt-0.11.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d1b36540d7aaf9b9101b3a6f376c8d8e9f7a9aec93ed05918f2c69d493ffef72"},
+ {file = "librt-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:efbb343ab2ce3540f4ecbe6315d677ed70f37cd9a72b1e58066c918ca83acbaa"},
+ {file = "librt-0.11.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0dd688aab3f7914d3e6e5e3554978e0383312fb8e771d84be008a35b9ee548"},
+ {file = "librt-0.11.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f5fb36b8c6c63fdcbb1d526d94c0d1331610d43f4118cc1beb4efef4f3faacb2"},
+ {file = "librt-0.11.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4a9a237d13addb93715b6fee74023d5ee3469b53fce527626c0e088aa585805f"},
+ {file = "librt-0.11.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ddd17bd87b2c56ddd60e546a7984a2e64c4e8eab92fb4cf3830a48ad5469d51"},
+ {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd43992b4473d42f12ff9e68326079f0696d9d4e6000e8f39a0238d482ba6ee2"},
+ {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:f8e3e8056dd674e279741485e2e512d6e9a751c7455809d0114e6ebf8d781085"},
+ {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c1f708d8ae9c56cf38a903c44297243d2ec83fd82b396b977e0144a3e76217e3"},
+ {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0add982e0e7b9fc14cf4b33789d5f13f66581889b88c2f58099f6ce8f92617bd"},
+ {file = "librt-0.11.0-cp314-cp314t-win32.whl", hash = "sha256:2b481d846ac894c4e8403c5fd0e87c5d11d6499e404b474602508a224ff531c8"},
+ {file = "librt-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:28edb433edde181112a908c78907af28f964eabc15f4dd16c9d66c834302677c"},
+ {file = "librt-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253"},
+ {file = "librt-0.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6bd72d903911d995ab666dbd1871f8b1e80925a699af8063fbf50053329fb05f"},
+ {file = "librt-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ef69ac715f3cd8e5cd252cb2aebfa72c015492aacc339d5d7bf8fef3c62c677"},
+ {file = "librt-0.11.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:624a40c4a4ad7773315c287276cd024509b2c66ff5904f504bfc08d2c70293ab"},
+ {file = "librt-0.11.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:41dc19fe150b69716c8ece4f76773a9e8813fe3e35e032a58b4d46423fb8d7c0"},
+ {file = "librt-0.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4e8bd98ea9c47ae90b319a087ab28dac493f1ffbc1ecd1f28fcdbf3b7e1108d1"},
+ {file = "librt-0.11.0-cp39-cp39-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84308fc49423ce6475d1c5d1985cd69a8ca9f0325fc7d5f81bb690a3f3625d4e"},
+ {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ff0fbaf5f44a21beeb0110f2ab64f45135a9536a834b79c0d1ef018f2786bbfa"},
+ {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:9c028a9442a18e266955d364ce42259136e79a7ba14d773e0d778d5f70cd56f1"},
+ {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9f1692105a02bcf853f355032a5fdc5494358ef83d8fd22d16de375c85cec3f5"},
+ {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7a80a71e1fda83cc752a9141e87aae7fef279538597564d670e9ce513f286192"},
+ {file = "librt-0.11.0-cp39-cp39-win32.whl", hash = "sha256:140695816ddf3c86eb972981a26f35efd871c44b0c3aed44c8cd01749386617f"},
+ {file = "librt-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:92f7ff819c197fc30473190a12c2856f325ac90aabfccbeb2072d28cc2e234e3"},
+ {file = "librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1"},
+]
+
[[package]]
name = "lxml"
version = "4.9.4"
@@ -1782,50 +1894,71 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"]
[[package]]
name = "mypy"
-version = "1.11.2"
+version = "1.20.2"
description = "Optional static typing for Python"
optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.10"
groups = ["dev", "lint"]
files = [
- {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"},
- {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"},
- {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"},
- {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"},
- {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"},
- {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"},
- {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"},
- {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"},
- {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"},
- {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"},
- {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"},
- {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"},
- {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"},
- {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"},
- {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"},
- {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"},
- {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"},
- {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"},
- {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"},
- {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"},
- {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"},
- {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"},
- {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"},
- {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"},
- {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"},
- {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"},
- {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"},
+ {file = "mypy-1.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cf5a4db6dca263010e2c7bff081c89383c72d187ba2cf4c44759aac970e2f0c4"},
+ {file = "mypy-1.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b0e817b518bff7facd7f85ea05b643ad8bdcce684cf29784987b0a7c8e1f997"},
+ {file = "mypy-1.20.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97d7b9a485b40f8ca425460e89bf1da2814625b2da627c0dcc6aa46c92631d14"},
+ {file = "mypy-1.20.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e1c12f6d2db3d78b909b5f77513c11eb7f2dd2782b96a3ab6dffc7d44575c99"},
+ {file = "mypy-1.20.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89dce27e142d25ffbc154c1819383b69f2e9234dc4ed4766f42e0e8cb264ab5c"},
+ {file = "mypy-1.20.2-cp310-cp310-win_amd64.whl", hash = "sha256:f376e37f9bf2a946872fc5fd1199c99310748e3c26c7a26683f13f8bdb756cbd"},
+ {file = "mypy-1.20.2-cp310-cp310-win_arm64.whl", hash = "sha256:6e2b469efd811707bc530fd1effef0f5d6eebcb7fe376affae69025da4b979a2"},
+ {file = "mypy-1.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4077797a273e56e8843d001e9dfe4ba10e33323d6ade647ff260e5cd97d9758c"},
+ {file = "mypy-1.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cdecf62abcc4292500d7858aeae87a1f8f1150f4c4dd08fb0b336ee79b2a6df3"},
+ {file = "mypy-1.20.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c566c3a88b6ece59b3d70f65bedef17304f48eb52ff040a6a18214e1917b3254"},
+ {file = "mypy-1.20.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0deb80d062b2479f2c87ae568f89845afc71d11bc41b04179e58165fd9f31e98"},
+ {file = "mypy-1.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bba9ad231e92a3e424b3e56b65aa17704993425bba97e302c832f9466bb85bac"},
+ {file = "mypy-1.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:baf593f2765fa3a6b1ef95807dbaa3d25b594f6a52adcc506a6b9cb115e1be67"},
+ {file = "mypy-1.20.2-cp311-cp311-win_arm64.whl", hash = "sha256:20175a1c0f49863946ec20b7f63255768058ac4f07d2b9ded6a6b46cfb5a9100"},
+ {file = "mypy-1.20.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4dbfcf869f6b0517f70cf0030ba6ea1d6645e132337a7d5204a18d8d5636c02b"},
+ {file = "mypy-1.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b6481b228d072315b053210b01ac320e1be243dc17f9e5887ef167f23f5fae4"},
+ {file = "mypy-1.20.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34397cdced6b90b836e38182076049fdb41424322e0b0728c946b0939ebdf9f6"},
+ {file = "mypy-1.20.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5da6976f20cae27059ea8d0c86e7cef3de720e04c4bb9ee18e3690fdb792066"},
+ {file = "mypy-1.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56908d7e08318d39f85b1f0c6cfd47b0cac1a130da677630dac0de3e0623e102"},
+ {file = "mypy-1.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:d52ad8d78522da1d308789df651ee5379088e77c76cb1994858d40a426b343b9"},
+ {file = "mypy-1.20.2-cp312-cp312-win_arm64.whl", hash = "sha256:785b08db19c9f214dc37d65f7c165d19a30fcecb48abfa30f31b01b5acaabb58"},
+ {file = "mypy-1.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:edfbfca868cdd6bd8d974a60f8a3682f5565d3f5c99b327640cedd24c4264026"},
+ {file = "mypy-1.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2877a02380adfcdbc69071a0f74d6e9dbbf593c0dc9d174e1f223ffd5281943"},
+ {file = "mypy-1.20.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7488448de6007cd5177c6cea0517ac33b4c0f5ee9b5e9f2be51ce75511a85517"},
+ {file = "mypy-1.20.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb9c2fa06887e21d6a3a868762acb82aec34e2c6fd0174064f27c93ede68ad15"},
+ {file = "mypy-1.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d56a78b646f2e3daa865bc70cd5ec5a46c50045801ca8ff17a0c43abc97e3ee"},
+ {file = "mypy-1.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:2a4102b03bb7481d9a91a6da8d174740c9c8c4401024684b9ca3b7cc5e49852f"},
+ {file = "mypy-1.20.2-cp313-cp313-win_arm64.whl", hash = "sha256:a95a9248b0c6fd933a442c03c3b113c3b61320086b88e2c444676d3fd1ca3330"},
+ {file = "mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30"},
+ {file = "mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924"},
+ {file = "mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb"},
+ {file = "mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc"},
+ {file = "mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558"},
+ {file = "mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8"},
+ {file = "mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3"},
+ {file = "mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609"},
+ {file = "mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2"},
+ {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c"},
+ {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744"},
+ {file = "mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6"},
+ {file = "mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec"},
+ {file = "mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382"},
+ {file = "mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563"},
+ {file = "mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665"},
]
[package.dependencies]
-mypy-extensions = ">=1.0.0"
+librt = {version = ">=0.8.0", markers = "platform_python_implementation != \"PyPy\""}
+mypy_extensions = ">=1.0.0"
+pathspec = ">=1.0.0"
tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
-typing-extensions = ">=4.6.0"
+typing_extensions = {version = ">=4.6.0", markers = "python_version < \"3.15\""}
[package.extras]
dmypy = ["psutil (>=4.0)"]
+faster-cache = ["orjson"]
install-types = ["pip"]
mypyc = ["setuptools (>=50)"]
+native-parser = ["ast-serialize (>=0.1.1,<1.0.0)"]
reports = ["lxml"]
[[package]]
@@ -2367,63 +2500,158 @@ files = [
[[package]]
name = "pydantic"
-version = "1.10.19"
-description = "Data validation and settings management using python type hints"
+version = "2.13.4"
+description = "Data validation using Python type hints"
optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "pydantic-1.10.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a415b9e95fa602b10808113967f72b2da8722061265d6af69268c111c254832d"},
- {file = "pydantic-1.10.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:11965f421f7eb026439d4eb7464e9182fe6d69c3d4d416e464a4485d1ba61ab6"},
- {file = "pydantic-1.10.19-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5bb81fcfc6d5bff62cd786cbd87480a11d23f16d5376ad2e057c02b3b44df96"},
- {file = "pydantic-1.10.19-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83ee8c9916689f8e6e7d90161e6663ac876be2efd32f61fdcfa3a15e87d4e413"},
- {file = "pydantic-1.10.19-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0399094464ae7f28482de22383e667625e38e1516d6b213176df1acdd0c477ea"},
- {file = "pydantic-1.10.19-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8b2cf5e26da84f2d2dee3f60a3f1782adedcee785567a19b68d0af7e1534bd1f"},
- {file = "pydantic-1.10.19-cp310-cp310-win_amd64.whl", hash = "sha256:1fc8cc264afaf47ae6a9bcbd36c018d0c6b89293835d7fb0e5e1a95898062d59"},
- {file = "pydantic-1.10.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d7a8a1dd68bac29f08f0a3147de1885f4dccec35d4ea926e6e637fac03cdb4b3"},
- {file = "pydantic-1.10.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07d00ca5ef0de65dd274005433ce2bb623730271d495a7d190a91c19c5679d34"},
- {file = "pydantic-1.10.19-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad57004e5d73aee36f1e25e4e73a4bc853b473a1c30f652dc8d86b0a987ffce3"},
- {file = "pydantic-1.10.19-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dce355fe7ae53e3090f7f5fa242423c3a7b53260747aa398b4b3aaf8b25f41c3"},
- {file = "pydantic-1.10.19-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0d32227ea9a3bf537a2273fd2fdb6d64ab4d9b83acd9e4e09310a777baaabb98"},
- {file = "pydantic-1.10.19-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e351df83d1c9cffa53d4e779009a093be70f1d5c6bb7068584086f6a19042526"},
- {file = "pydantic-1.10.19-cp311-cp311-win_amd64.whl", hash = "sha256:d8d72553d2f3f57ce547de4fa7dc8e3859927784ab2c88343f1fc1360ff17a08"},
- {file = "pydantic-1.10.19-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d5b5b7c6bafaef90cbb7dafcb225b763edd71d9e22489647ee7df49d6d341890"},
- {file = "pydantic-1.10.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:570ad0aeaf98b5e33ff41af75aba2ef6604ee25ce0431ecd734a28e74a208555"},
- {file = "pydantic-1.10.19-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0890fbd7fec9e151c7512941243d830b2d6076d5df159a2030952d480ab80a4e"},
- {file = "pydantic-1.10.19-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec5c44e6e9eac5128a9bfd21610df3b8c6b17343285cc185105686888dc81206"},
- {file = "pydantic-1.10.19-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6eb56074b11a696e0b66c7181da682e88c00e5cebe6570af8013fcae5e63e186"},
- {file = "pydantic-1.10.19-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9d7d48fbc5289efd23982a0d68e973a1f37d49064ccd36d86de4543aff21e086"},
- {file = "pydantic-1.10.19-cp312-cp312-win_amd64.whl", hash = "sha256:fd34012691fbd4e67bdf4accb1f0682342101015b78327eaae3543583fcd451e"},
- {file = "pydantic-1.10.19-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a5d5b877c7d3d9e17399571a8ab042081d22fe6904416a8b20f8af5909e6c8f"},
- {file = "pydantic-1.10.19-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c46f58ef2df958ed2ea7437a8be0897d5efe9ee480818405338c7da88186fb3"},
- {file = "pydantic-1.10.19-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d8a38a44bb6a15810084316ed69c854a7c06e0c99c5429f1d664ad52cec353c"},
- {file = "pydantic-1.10.19-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a82746c6d6e91ca17e75f7f333ed41d70fce93af520a8437821dec3ee52dfb10"},
- {file = "pydantic-1.10.19-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:566bebdbe6bc0ac593fa0f67d62febbad9f8be5433f686dc56401ba4aab034e3"},
- {file = "pydantic-1.10.19-cp37-cp37m-win_amd64.whl", hash = "sha256:22a1794e01591884741be56c6fba157c4e99dcc9244beb5a87bd4aa54b84ea8b"},
- {file = "pydantic-1.10.19-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:076c49e24b73d346c45f9282d00dbfc16eef7ae27c970583d499f11110d9e5b0"},
- {file = "pydantic-1.10.19-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d4320510682d5a6c88766b2a286d03b87bd3562bf8d78c73d63bab04b21e7b4"},
- {file = "pydantic-1.10.19-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e66aa0fa7f8aa9d0a620361834f6eb60d01d3e9cea23ca1a92cda99e6f61dac"},
- {file = "pydantic-1.10.19-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d216f8d0484d88ab72ab45d699ac669fe031275e3fa6553e3804e69485449fa0"},
- {file = "pydantic-1.10.19-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9f28a81978e936136c44e6a70c65bde7548d87f3807260f73aeffbf76fb94c2f"},
- {file = "pydantic-1.10.19-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d3449633c207ec3d2d672eedb3edbe753e29bd4e22d2e42a37a2c1406564c20f"},
- {file = "pydantic-1.10.19-cp38-cp38-win_amd64.whl", hash = "sha256:7ea24e8614f541d69ea72759ff635df0e612b7dc9d264d43f51364df310081a3"},
- {file = "pydantic-1.10.19-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:573254d844f3e64093f72fcd922561d9c5696821ff0900a0db989d8c06ab0c25"},
- {file = "pydantic-1.10.19-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ff09600cebe957ecbb4a27496fe34c1d449e7957ed20a202d5029a71a8af2e35"},
- {file = "pydantic-1.10.19-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4739c206bfb6bb2bdc78dcd40bfcebb2361add4ceac6d170e741bb914e9eff0f"},
- {file = "pydantic-1.10.19-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bfb5b378b78229119d66ced6adac2e933c67a0aa1d0a7adffbe432f3ec14ce4"},
- {file = "pydantic-1.10.19-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7f31742c95e3f9443b8c6fa07c119623e61d76603be9c0d390bcf7e888acabcb"},
- {file = "pydantic-1.10.19-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c6444368b651a14c2ce2fb22145e1496f7ab23cbdb978590d47c8d34a7bc0289"},
- {file = "pydantic-1.10.19-cp39-cp39-win_amd64.whl", hash = "sha256:945407f4d08cd12485757a281fca0e5b41408606228612f421aa4ea1b63a095d"},
- {file = "pydantic-1.10.19-py3-none-any.whl", hash = "sha256:2206a1752d9fac011e95ca83926a269fb0ef5536f7e053966d058316e24d929f"},
- {file = "pydantic-1.10.19.tar.gz", hash = "sha256:fea36c2065b7a1d28c6819cc2e93387b43dd5d3cf5a1e82d8132ee23f36d1f10"},
+ {file = "pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba"},
+ {file = "pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6"},
]
[package.dependencies]
-typing-extensions = ">=4.2.0"
+annotated-types = ">=0.6.0"
+pydantic-core = "2.46.4"
+typing-extensions = ">=4.14.1"
+typing-inspection = ">=0.4.2"
[package.extras]
-dotenv = ["python-dotenv (>=0.10.4)"]
-email = ["email-validator (>=1.0.3)"]
+email = ["email-validator (>=2.0.0)"]
+timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""]
+
+[[package]]
+name = "pydantic-core"
+version = "2.46.4"
+description = "Core functionality for Pydantic validation and serialization"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+ {file = "pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d"},
+ {file = "pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33"},
+ {file = "pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89"},
+ {file = "pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292"},
+ {file = "pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac"},
+ {file = "pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5"},
+ {file = "pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_31_riscv64.whl", hash = "sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-win32.whl", hash = "sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1"},
+ {file = "pydantic_core-2.46.4-cp39-cp39-win_amd64.whl", hash = "sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac"},
+ {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c"},
+ {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b"},
+ {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b"},
+ {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea"},
+ {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7"},
+ {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df"},
+ {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526"},
+ {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc"},
+ {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983"},
+ {file = "pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1"},
+]
+
+[package.dependencies]
+typing-extensions = ">=4.14.1"
[[package]]
name = "pygments"
@@ -2991,6 +3219,21 @@ files = [
]
markers = {docs = "python_version == \"3.10\"", test = "python_version == \"3.10\""}
+[[package]]
+name = "typing-inspection"
+version = "0.4.2"
+description = "Runtime typing introspection tools"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+ {file = "typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"},
+ {file = "typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"},
+]
+
+[package.dependencies]
+typing-extensions = ">=4.12.0"
+
[[package]]
name = "tzdata"
version = "2026.2"
@@ -3298,4 +3541,4 @@ type = ["pytest-mypy"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.13"
-content-hash = "dc39317cbf4102e2fd18a441031ed03d8fb788fb0b54c88e6a648ab301ba1e55"
+content-hash = "793e65e9dcefec411b9423ed3587c05121529b422f9299ac99cd157c7dec7c01"
diff --git a/pyproject.toml b/pyproject.toml
index 59c1d2d..64e4bc8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,7 @@ openpyxl = "3.1.5"
pandas = "2.3.3"
polars = "0.20.31"
pyarrow = "17.0.0"
-pydantic = "1.10.19"
+pydantic = "2.13.4"
pyspark = "3.5.2"
typing_extensions = "4.15.0"
@@ -80,7 +80,8 @@ black = "24.3.0"
astroid = "3.3.9"
isort = "5.13.2"
pylint = "3.3.9"
-mypy = "1.11.2"
+mypy = "1.20.2"
+librt = "0.11.0" # mypy dependency
boto3-stubs = {extras = ["essential"], version = "1.26.72"}
botocore-stubs = "1.29.72"
pandas-stubs = "1.2.0.62"
diff --git a/src/dve/core_engine/backends/base/auditing.py b/src/dve/core_engine/backends/base/auditing.py
index d120fcb..c921cd9 100644
--- a/src/dve/core_engine/backends/base/auditing.py
+++ b/src/dve/core_engine/backends/base/auditing.py
@@ -14,7 +14,7 @@
from types import TracebackType
from typing import Any, ClassVar, Generic, Optional, TypeVar, Union
-from pydantic import ValidationError, validate_arguments
+from pydantic import ValidationError, validate_call
from typing_extensions import Literal, get_origin
from dve.core_engine.models import (
@@ -98,8 +98,8 @@ def __init__(self, name: str, record_type: type[AuditRecord]):
def schema(self) -> dict[str, type]:
"""Determine python schema of auditor"""
return {
- fld: str if get_origin(mdl.type_) == Literal else mdl.type_
- for fld, mdl in self._record_type.__fields__.items()
+ fld: str if get_origin(mdl.annotation) == Literal else mdl.annotation # type: ignore
+ for fld, mdl in self._record_type.model_fields.items()
}
@staticmethod
@@ -195,7 +195,7 @@ def conv_to_iterable(recs: Union[AuditorType, AuditReturnType]) -> Iterable[dict
"""Convert AuditReturnType to iterable of dictionaries"""
raise NotImplementedError()
- @validate_arguments
+ @validate_call
def add_processing_records(self, processing_records: list[ProcessingStatusRecord]):
"""Add an entry to the processing_status auditor."""
if self.pool:
@@ -207,7 +207,7 @@ def add_processing_records(self, processing_records: list[ProcessingStatusRecord
records=[dict(rec) for rec in processing_records]
)
- @validate_arguments
+ @validate_call
def add_submission_statistics_records(self, sub_stats: list[SubmissionStatisticsRecord]):
"""Add an entry to the submission statistics auditor."""
if self.pool:
@@ -217,7 +217,7 @@ def add_submission_statistics_records(self, sub_stats: list[SubmissionStatistics
)
return self._submission_statistics.add_records(records=[dict(rec) for rec in sub_stats])
- @validate_arguments
+ @validate_call
def add_transfer_records(self, transfer_records: list[TransferRecord]):
"""Add an entry to the transfers auditor"""
if self.pool:
@@ -226,7 +226,7 @@ def add_transfer_records(self, transfer_records: list[TransferRecord]):
)
return self._transfers.add_records(records=[dict(rec) for rec in transfer_records])
- @validate_arguments
+ @validate_call
def add_new_submissions(
self,
submissions: list[SubmissionMetadata],
@@ -249,7 +249,7 @@ def add_new_submissions(
processing_status="received",
job_run_id=job_run_id,
**ts_info,
- ).dict(),
+ ).model_dump(),
}
processing_status_recs.append(processing_rec)
if sub_info:
diff --git a/src/dve/core_engine/backends/implementations/duckdb/auditing.py b/src/dve/core_engine/backends/implementations/duckdb/auditing.py
index 3124c6d..b3c8b75 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/auditing.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/auditing.py
@@ -14,10 +14,10 @@
OrderCriteria,
)
from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
- PYTHON_TYPE_TO_DUCKDB_TYPE,
+ get_duckdb_type_from_annotation,
table_exists,
)
-from dve.core_engine.backends.utilities import PYTHON_TYPE_TO_POLARS_TYPE
+from dve.core_engine.backends.utilities import get_polars_type_from_annotation
from dve.core_engine.models import (
AuditRecord,
ProcessingStatusRecord,
@@ -62,7 +62,10 @@ def ddb_create_table_sql(self) -> str:
"""Generate create table sql script for auditor"""
_sql_expression = f"CREATE TABLE {self._name} ("
_sql_expression += ", ".join(
- [f"{fld} {PYTHON_TYPE_TO_DUCKDB_TYPE.get(dtype)}" for fld, dtype in self.schema.items()]
+ [
+ f"{fld} {get_duckdb_type_from_annotation(dtype)}"
+ for fld, dtype in self.schema.items()
+ ]
)
_sql_expression += ")"
return _sql_expression
@@ -70,10 +73,7 @@ def ddb_create_table_sql(self) -> str:
@property
def polars_schema(self) -> dict[str, PolarsType]:
"""Get polars dataframe schema for auditor"""
- return {
- fld: PYTHON_TYPE_TO_POLARS_TYPE.get(dtype, pl.Utf8) # type: ignore
- for fld, dtype in self.schema.items()
- }
+ return {fld: get_polars_type_from_annotation(dtype) for fld, dtype in self.schema.items()}
def get_relation(self) -> DuckDBPyRelation:
"""Get a relation to interact with the auditor duckdb table"""
@@ -106,7 +106,7 @@ def conv_to_entity(self, recs: list[AuditRecord]) -> DuckDBPyRelation:
"""Convert a list of audit records to a relation"""
# pylint: disable=W0612
rec_df = pl.DataFrame( # type: ignore
- [rec.dict() for rec in recs],
+ [rec.model_dump() for rec in recs],
schema=self.polars_schema,
)
return self._connection.sql("select * from rec_df")
diff --git a/src/dve/core_engine/backends/implementations/duckdb/contract.py b/src/dve/core_engine/backends/implementations/duckdb/contract.py
index 3595716..d9bb9bc 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/contract.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/contract.py
@@ -14,7 +14,6 @@
from duckdb.typing import DuckDBPyType
from polars.datatypes.classes import DataTypeClass as PolarsType
from pydantic import BaseModel
-from pydantic.fields import ModelField
import dve.parser.file_handling as fh
from dve.common.error_utils import (
@@ -96,8 +95,8 @@ def create_entity_from_py_iterator( # pylint: disable=unused-argument
) -> DuckDBPyRelation:
"""Create DuckDB Relation from iterator of records"""
polars_schema: dict[str, PolarsType] = {
- fld.name: get_polars_type_from_annotation(fld.type_)
- for fld in stringify_model(schema).__fields__.values()
+ name: get_polars_type_from_annotation(fld.annotation)
+ for name, fld in stringify_model(schema).model_fields.items()
}
_lazy_df = pl.LazyFrame(records, polars_schema) # type: ignore # pylint: disable=unused-variable
return self._connection.sql("select * from _lazy_df")
@@ -130,17 +129,15 @@ def apply_data_contract(
) as msg_writer:
for entity_name, relation in entities.items():
# get dtypes for all fields -> python data types or use with relation
- entity_fields: dict[str, ModelField] = contract_metadata.schemas[
- entity_name
- ].__fields__
+ entity_fields = contract_metadata.schemas[entity_name].model_fields
ddb_schema: dict[str, DuckDBPyType] = {
- fld.name: get_duckdb_type_from_annotation(fld.annotation)
- for fld in entity_fields.values()
+ name: get_duckdb_type_from_annotation(fld.annotation)
+ for name, fld in entity_fields.items()
}
ddb_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int)
polars_schema: dict[str, PolarsType] = {
- fld.name: get_polars_type_from_annotation(fld.annotation)
- for fld in entity_fields.values()
+ name: get_polars_type_from_annotation(fld.annotation)
+ for name, fld in entity_fields.items()
}
polars_schema[RECORD_INDEX_COLUMN_NAME] = get_polars_type_from_annotation(int)
if relation_is_empty(relation):
diff --git a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py
index 627822b..724a5f5 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py
@@ -7,7 +7,7 @@
from datetime import date, datetime, time
from decimal import Decimal
from pathlib import Path
-from typing import Any, ClassVar, Union
+from typing import Any, ClassVar, Literal, Union
from urllib.parse import urlparse
import duckdb.typing as ddbtyp
@@ -125,8 +125,8 @@ def get_duckdb_type_from_annotation(type_annotation: Any) -> DuckDBPyType:
'optional' wrapper and return the inner type
- A subclass of `typing.TypedDict` with values typed using supported types. This
will parse the value types as Polars types and return a duckdb STRUCT.
- - A dataclass or `pydantic.main.ModelMetaClass` with values typed using supported types.
- This will parse the field types as Polars types and return a duckdb STRUCT.
+ - A dataclass or `pydantic.BaseModel` with values typed using supported types.
+ This will parse the field types as duckdb types and return a duckdb STRUCT.
- Any supported type, with a `typing_extensions.Annotated` wrapper.
Any `ClassVar` types within `TypedDict`s, dataclasses, or `pydantic` models will be
@@ -135,6 +135,14 @@ def get_duckdb_type_from_annotation(type_annotation: Any) -> DuckDBPyType:
"""
type_origin = get_origin(type_annotation)
+ if type_origin is Literal:
+ ddb_types = [get_duckdb_type_from_annotation(type(t)) for t in get_args(type_annotation)]
+ if not ddb_types or not all(t == ddb_types[0] for t in ddb_types):
+ raise ValueError(
+ f"Unable to determine a single concrete type for Literal. Got {type_annotation!r}"
+ )
+ return ddb_types[0]
+
# An `Optional` or `Union` type, check to ensure non-heterogenity.
if type_origin is Union:
python_type = _get_non_heterogenous_type(get_args(type_annotation))
diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py
index 17d7635..7d8201d 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py
@@ -110,8 +110,8 @@ def read_to_relation( # pylint: disable=unused-argument
}
ddb_schema: dict[str, SQLType] = {
- fld.name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore
- for fld in schema.__fields__.values()
+ name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore
+ for name, fld in schema.model_fields.items()
}
reader_options["columns"] = ddb_schema
@@ -154,8 +154,8 @@ def read_to_relation( # pylint: disable=unused-argument
}
polars_types = {
- fld.name: get_polars_type_from_annotation(fld.annotation) # type: ignore
- for fld in schema.__fields__.values()
+ name: get_polars_type_from_annotation(fld.annotation) # type: ignore
+ for name, fld in schema.model_fields.items()
}
reader_options["dtypes"] = polars_types
diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py
index cf0fa82..e316593 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py
@@ -48,8 +48,8 @@ def read_to_relation( # pylint: disable=unused-argument
"""Returns a relation object from the source json"""
ddb_schema: dict[str, SQLType] = {
- fld.name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore
- for fld in schema.__fields__.values()
+ name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore
+ for name, fld in schema.model_fields.items()
}
return self.add_record_index(
diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py
index c63e464..678047d 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py
@@ -41,8 +41,8 @@ def read_to_relation(self, resource: URI, entity_name: str, schema: type[BaseMod
)
polars_schema: dict[str, pl.DataType] = { # type: ignore
- fld.name: get_polars_type_from_annotation(fld.annotation)
- for fld in stringify_model(schema).__fields__.values()
+ name: get_polars_type_from_annotation(fld.annotation)
+ for name, fld in stringify_model(schema).model_fields.items()
}
_lazy_frame = self.add_record_index(
diff --git a/src/dve/core_engine/backends/implementations/spark/auditing.py b/src/dve/core_engine/backends/implementations/spark/auditing.py
index c050a17..3f50721 100644
--- a/src/dve/core_engine/backends/implementations/spark/auditing.py
+++ b/src/dve/core_engine/backends/implementations/spark/auditing.py
@@ -116,7 +116,7 @@ def conv_to_records(self, recs: DataFrame) -> Iterable[AuditRecord]:
def conv_to_entity(self, recs: list[AuditRecord]) -> DataFrame:
"""Convert the dataframe to an iterable of the related audit record"""
return self._spark.createDataFrame( # type: ignore
- [rec.dict() for rec in recs], schema=self.spark_schema
+ [rec.model_dump() for rec in recs], schema=self.spark_schema
)
def add_records(self, records: Iterable[dict[str, Any]]):
diff --git a/src/dve/core_engine/backends/implementations/spark/backend.py b/src/dve/core_engine/backends/implementations/spark/backend.py
index 126e07a..abd1540 100644
--- a/src/dve/core_engine/backends/implementations/spark/backend.py
+++ b/src/dve/core_engine/backends/implementations/spark/backend.py
@@ -92,5 +92,5 @@ def write_entities_to_parquet(
def convert_submission_info(self, submission_info: SubmissionInfo) -> DataFrame:
return self.spark_session.createDataFrame( # type: ignore
- [submission_info.dict()], schema=get_type_from_annotation(type(submission_info))
+ [submission_info.model_dump()], schema=get_type_from_annotation(type(submission_info))
)
diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
index ced985a..92703cc 100644
--- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
+++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
@@ -12,11 +12,10 @@
from dataclasses import dataclass, is_dataclass
from decimal import Decimal
from functools import wraps
-from typing import Any, ClassVar, Optional, TypeVar, Union, overload
+from typing import Any, ClassVar, Literal, Optional, TypeVar, Union, overload
from delta.exceptions import ConcurrentAppendException, DeltaConcurrentModificationException
from pydantic import BaseModel
-from pydantic.types import ConstrainedDecimal
from pyspark.sql import DataFrame, Row, SparkSession
from pyspark.sql import functions as sf
from pyspark.sql import types as st
@@ -49,6 +48,7 @@
"""A wrapped function (Spark UDF) taking four args."""
+# TODO - lets see if we can bin this off as it's a bit overkill
@dataclass(frozen=True)
class DecimalConfig:
"""Configuration for a Python decimal to enable it to be mapped to a
@@ -61,13 +61,13 @@ class DecimalConfig:
"""
- precision: int = 38
+ max_digits: int = 38
"""
The precision of the decimal. This is the total number of digits in the
decimal.
"""
- scale: int = 18
+ decimal_places: int = 18
"""
The scale of the decimal. This is the number of digits to the right of the
decimal point.
@@ -75,10 +75,12 @@ class DecimalConfig:
"""
def __post_init__(self):
- if not 0 < self.precision <= 38:
- raise ValueError("Precision must be between 1 and 38 (inclusive)")
- if not 0 <= self.scale <= self.precision:
- raise ValueError("Scale must be between 0 and the precision (inclusive)")
+ if not 0 < self.max_digits <= 38:
+ raise ValueError("Max digits must be between 1 and 38 (inclusive)")
+ if not 0 <= self.decimal_places <= self.max_digits:
+ raise ValueError(
+ "Decimal Places must be between 0 and the specified number of digits (inclusive)"
+ )
DEFAULT_DECIMAL_CONFIG = DecimalConfig()
@@ -93,7 +95,9 @@ def __post_init__(self):
bytes: st.BinaryType(),
dt.date: st.DateType(),
dt.datetime: st.TimestampType(),
- Decimal: st.DecimalType(DEFAULT_DECIMAL_CONFIG.precision, DEFAULT_DECIMAL_CONFIG.scale),
+ Decimal: st.DecimalType(
+ DEFAULT_DECIMAL_CONFIG.max_digits, DEFAULT_DECIMAL_CONFIG.decimal_places
+ ),
}
"""A mapping of Python types to the equivalent Spark types."""
@@ -146,7 +150,7 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType:
'optional' wrapper and return the inner type (Spark types are all nullable).
- A subclass of `typing.TypedDict` with values typed using supported types. This
will parse the value types as Spark types and return a Spark `StructType`.
- - A dataclass or `pydantic.main.ModelMetaClass` with values typed using supported types.
+ - A dataclass or `pydantic.BaseModel` with values typed using supported types.
This will parse the field types as Spark types and return a Spark `StructType`.
- Any supported type, with a `typing_extensions.Annotated` wrapper.
- A `decimal.Decimal` wrapped with `typing_extensions.Annotated` with a `DecimalConfig`
@@ -160,6 +164,14 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType:
"""
type_origin = get_origin(type_annotation)
+ if type_origin is Literal:
+ types = [get_type_from_annotation(type(t)) for t in get_args(type_annotation)]
+ if not types or not all(t == types[0] for t in types):
+ raise ValueError(
+ f"Unable to determine a single concrete type for Literal. Got {type_annotation!r}"
+ )
+ return types[0]
+
# An `Optional` or `Union` type, check to ensure non-heterogenity.
if type_origin is Union:
python_type = _get_non_heterogenous_type(get_args(type_annotation))
@@ -176,13 +188,13 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType:
if python_type is not Decimal:
return get_type_from_annotation(python_type)
- try: # Grab the decimal configuration from the list of other args.
- configuration: DecimalConfig = next(
- filter(lambda config: isinstance(config, DecimalConfig), other_args)
- )
- except StopIteration:
+ config_options = [arg for arg in other_args if hasattr(arg, "max_digits")]
+ if config_options:
+ configuration = config_options[0]
+ else:
configuration = DEFAULT_DECIMAL_CONFIG
- return st.DecimalType(configuration.precision, configuration.scale)
+
+ return st.DecimalType(configuration.max_digits, configuration.decimal_places)
# Ensure that we have a concrete type at this point.
if not isinstance(type_annotation, type):
@@ -216,11 +228,6 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType:
return st.StructType(fields)
- if issubclass(type_annotation, ConstrainedDecimal):
- precision = int(type_annotation.max_digits or 38)
- scale = int(type_annotation.decimal_places or precision)
- return st.DecimalType(precision, scale)
-
if type_annotation is list:
raise ValueError(
f"list must have type annotation (e.g. `list[str]`), got {type_annotation!r}"
diff --git a/src/dve/core_engine/backends/implementations/spark/utilities.py b/src/dve/core_engine/backends/implementations/spark/utilities.py
index 5eca158..399d355 100644
--- a/src/dve/core_engine/backends/implementations/spark/utilities.py
+++ b/src/dve/core_engine/backends/implementations/spark/utilities.py
@@ -110,7 +110,7 @@ class PydanticCompatibleJSONEncoder(JSONEncoder):
def default(self, o: Any) -> Any:
"""Sets the format for given types for json encoding"""
if isinstance(o, BaseModel):
- return o.dict()
+ return o.model_dump()
if isinstance(o, dt.date):
return o.isoformat()
return super().default(o)
diff --git a/src/dve/core_engine/backends/metadata/contract.py b/src/dve/core_engine/backends/metadata/contract.py
index 234a1e9..e3eb0c0 100644
--- a/src/dve/core_engine/backends/metadata/contract.py
+++ b/src/dve/core_engine/backends/metadata/contract.py
@@ -2,7 +2,7 @@
from typing import Any
-from pydantic import BaseModel, PrivateAttr, root_validator
+from pydantic import BaseModel, PrivateAttr, model_validator
from dve.core_engine.type_hints import EntityName, ReportingFields
from dve.core_engine.validation import RowValidator
@@ -44,10 +44,10 @@ def schemas(self) -> dict[EntityName, type[BaseModel]]:
"""The per-entity schemas, as pydantic models."""
if not self._schemas:
for entity_name, validator in self.validators.items():
- self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137
+ self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137
return self._schemas.copy() # pylint: disable=E1101
- @root_validator(allow_reuse=True)
+ @model_validator(mode="before")
@classmethod
def _ensure_entities_complete(cls, values: dict[str, dict[EntityName, Any]]):
"""Ensure the entities in 'readers' and 'validators' are the same."""
diff --git a/src/dve/core_engine/backends/metadata/reporting.py b/src/dve/core_engine/backends/metadata/reporting.py
index 3989a13..6385f37 100644
--- a/src/dve/core_engine/backends/metadata/reporting.py
+++ b/src/dve/core_engine/backends/metadata/reporting.py
@@ -5,7 +5,7 @@
from collections.abc import Callable
from typing import Any, ClassVar, Optional, Union
-from pydantic import BaseModel, root_validator, validate_arguments
+from pydantic import BaseModel, model_validator, validate_call
from typing_extensions import Literal
from dve.core_engine.templating import template_object
@@ -124,8 +124,10 @@ def template(
variables.update(local_variables)
else:
variables = local_variables
- templated = template_object(self.dict(exclude=self.UNTEMPLATED_FIELDS), variables, "jinja")
- templated.update(self.dict(include=self.UNTEMPLATED_FIELDS))
+ templated = template_object(
+ self.model_dump(exclude=self.UNTEMPLATED_FIELDS), variables, "jinja"
+ )
+ templated.update(self.model_dump(include=self.UNTEMPLATED_FIELDS))
return type_(**templated)
@@ -269,7 +271,7 @@ class LegacyReportingConfig(BaseReportingConfig):
legacy_is_informational: Optional[Union[bool, str]] = None
"""DEPRECATED: The legacy 'is_informational' flag."""
- @root_validator(allow_reuse=True, skip_on_failure=True)
+ @model_validator(mode="before")
@classmethod
def _ensure_only_one_reporting_config(cls, values: dict[str, Any]) -> dict[str, Any]:
"""Ensure only the modern or legacy location is populated."""
@@ -283,7 +285,7 @@ def _ensure_only_one_reporting_config(cls, values: dict[str, Any]) -> dict[str,
)
return values
- @root_validator(allow_reuse=True, skip_on_failure=True)
+ @model_validator(mode="before")
@classmethod
def _ensure_only_one_error_type_config(cls, values: dict[str, Any]) -> dict[str, Any]:
"""Ensure only the modern or legacy error type is populated."""
@@ -300,7 +302,7 @@ def _ensure_only_one_error_type_config(cls, values: dict[str, Any]) -> dict[str,
return values
@staticmethod
- @validate_arguments
+ @validate_call
def _convert_legacy_emit_value(
failure_type: Literal["record", "submission", "integrity", "group"], is_informational: bool
) -> str:
@@ -319,7 +321,7 @@ def _convert_legacy_emit_value(
return emit
@staticmethod
- @validate_arguments
+ @validate_call
def _convert_legacy_reporting_fields(
error_location: Optional[str] = None, reporting_field: Union[str, list[str], None] = None
) -> Optional[str]:
diff --git a/src/dve/core_engine/backends/metadata/rules.py b/src/dve/core_engine/backends/metadata/rules.py
index 7bc0353..f3a6305 100644
--- a/src/dve/core_engine/backends/metadata/rules.py
+++ b/src/dve/core_engine/backends/metadata/rules.py
@@ -1,11 +1,13 @@
"""Metadata classes for rule steps."""
+from __future__ import annotations
+
import warnings
from abc import ABCMeta, abstractmethod
from collections.abc import Iterator, Sequence
from typing import Any, ClassVar, Optional, TypeVar, Union
-from pydantic import BaseModel, Extra, Field, root_validator, validate_arguments, validator
+from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator
from typing_extensions import Literal
from dve.core_engine.backends.base.reference_data import ReferenceConfigUnion
@@ -72,16 +74,15 @@ def __repr__(self) -> str:
else:
components.append(f"rule=Rule(name={self.rule!r}, ...)")
- for key, value in self.dict(exclude={"rule"}).items():
+ for key, value in self.model_dump(exclude={"rule"}).items():
components.append(f"{key}={value!r}")
return f"ParentMetadata({', '.join(components)})"
- class Config: # pylint: disable=too-few-public-methods
- """`pydantic configuration options`"""
-
- frozen = True
- extra = Extra.forbid
+ model_config = {
+ "frozen": True,
+ "extra": "allow",
+ }
# pylint: disable=too-few-public-methods
@@ -98,11 +99,10 @@ class AbstractStep(BaseModel, metaclass=ABCMeta):
UNTEMPLATED_KEYS: ClassVar[set[str]] = {"id", "description", "parent"}
"""A set of aliases which are exempted from templating."""
- class Config: # pylint: disable=too-few-public-methods
- """`pydantic configuration options`"""
-
- frozen = True
- extra = Extra.forbid
+ model_config = {
+ "frozen": True,
+ "extra": "allow",
+ }
def __repr_args__(self) -> Sequence[tuple[Optional[str], Any]]:
# Exclude nulls from 'repr' for conciseness.
@@ -149,7 +149,7 @@ def template(
def __str__(self): # pydantic's default __str__ strips the model name.
return super().__repr__()
- @root_validator(pre=True)
+ @model_validator(mode="before")
@classmethod
def _warn_for_deprecated_aliases(cls, values: dict[str, JSONable]) -> dict[str, JSONable]:
for deprecated_name, replacement in (
@@ -375,27 +375,19 @@ class Aggregation(BaseStep):
agg_function: Optional[Alias] = None
"""The aggregate function to apply to the agg_columns (for duckdb backend)"""
- @validator("pivot_values")
+ @field_validator("pivot_values")
@classmethod
- def _ensure_column_if_values(
- cls,
- value: Optional[Any],
- values: dict[str, Any],
- ):
+ def _ensure_column_if_values(cls, value: Optional[Any], info: ValidationInfo):
"""Ensure that `pivot_column` is not null if pivot values are provided."""
- if value and not values["pivot_column"]:
+ if value and not info.data["pivot_column"]:
raise ValueError("`pivot_values` specified, but no `pivot_column`")
return value
- @validator("agg_function")
+ @field_validator("agg_function")
@classmethod
- def _ensure_column_if_function(
- cls,
- agg_function: Optional[Any],
- values: dict[str, Any],
- ):
+ def _ensure_column_if_function(cls, agg_function: Optional[Any], info: ValidationInfo):
"""Ensure that `pivot_column` is not null if pivot values are provided."""
- if agg_function and not values["agg_columns"]:
+ if agg_function and not info.data["agg_columns"]:
raise ValueError("`agg_function` specified, but no `agg_columns`")
return agg_function
@@ -582,7 +574,7 @@ def __str__(self): # pydantic's default __str__ strips the model name.
return super().__repr__()
@classmethod
- @validate_arguments
+ # @validate_call #TODO - removed for now as it's broken in pydantic v2
def from_step_list(cls, name: str, steps: list[Step]):
"""Load the rule from a single step list."""
pre_sync_steps: list[AbstractStep] = []
@@ -711,7 +703,7 @@ class RuleMetadata(BaseModel):
"""
- @root_validator()
+ @model_validator(mode="before")
@classmethod
def _ensure_locals_same_length_as_rules(cls, values: dict[str, list[Any]]):
"""Ensure that if 'local_variables' is provided, it's the same length as 'rules'."""
@@ -734,4 +726,4 @@ def __iter__(self) -> Iterator[tuple[Rule, TemplateVariables]]: # type: ignore
yield from zip(self.rules, self.local_variables)
-ParentMetadata.update_forward_refs()
+ParentMetadata.model_rebuild()
diff --git a/src/dve/core_engine/backends/readers/csv.py b/src/dve/core_engine/backends/readers/csv.py
index edd6bf0..055458a 100644
--- a/src/dve/core_engine/backends/readers/csv.py
+++ b/src/dve/core_engine/backends/readers/csv.py
@@ -196,7 +196,7 @@ def read_to_py_iterator(
if get_content_length(resource) == 0:
raise EmptyFileError(f"File at {resource!r} is empty")
- field_names = list(schema.__fields__.keys())
+ field_names = list(schema.model_fields.keys())
with open_stream(resource, "r", self.encoding) as stream:
reader = csv.DictReader(
stream,
@@ -223,8 +223,8 @@ def write_parquet( # type: ignore
target_location = file_uri_to_local_path(target_location).as_posix()
if schema:
polars_schema: dict[str, pl.DataType] = { # type: ignore
- fld.name: get_polars_type_from_annotation(fld.annotation)
- for fld in stringify_model(schema).__fields__.values()
+ name: get_polars_type_from_annotation(fld.annotation)
+ for name, fld in stringify_model(schema).model_fields.items()
}
polars_schema[RECORD_INDEX_COLUMN_NAME] = get_polars_type_from_annotation(int)
diff --git a/src/dve/core_engine/backends/readers/utilities.py b/src/dve/core_engine/backends/readers/utilities.py
index 642c0b2..6b103b6 100644
--- a/src/dve/core_engine/backends/readers/utilities.py
+++ b/src/dve/core_engine/backends/readers/utilities.py
@@ -17,5 +17,5 @@ def check_csv_header_expected(
"""Check the header of a CSV matches the expected fields"""
with open_stream(resource) as fle:
header_fields = fle.readline().rstrip().replace(quote_char, "").split(delimiter)
- expected_fields = expected_schema.__fields__.keys()
+ expected_fields = expected_schema.model_fields.keys()
return set(expected_fields).difference(header_fields)
diff --git a/src/dve/core_engine/backends/readers/xml.py b/src/dve/core_engine/backends/readers/xml.py
index 4620402..71a076e 100644
--- a/src/dve/core_engine/backends/readers/xml.py
+++ b/src/dve/core_engine/backends/readers/xml.py
@@ -13,7 +13,12 @@
from dve.core_engine.backends.base.reader import BaseFileReader
from dve.core_engine.backends.exceptions import EmptyFileError
from dve.core_engine.backends.readers.xml_linting import run_xmllint
-from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
+from dve.core_engine.backends.utilities import (
+ get_polars_type_from_annotation,
+ is_field_complex,
+ is_type_complex,
+ stringify_model,
+)
from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
from dve.core_engine.loggers import get_logger
from dve.core_engine.message import FeedbackMessage
@@ -43,6 +48,15 @@ def _strip_annotated(annotation: Any) -> Any:
return get_args(annotation)[0]
+def _strip_optional(annotation: Any) -> Any:
+ """Strip Optional type from a type"""
+ if hasattr(annotation, "_name"):
+ if annotation._name == "Optional": # pylint: disable=W0212
+ python_type, _default = get_args(annotation)
+ return python_type
+ return annotation
+
+
def create_template_row(schema: type[BaseModel]) -> dict[str, Any]:
"""Create a template row from a schema. A template row is essentially the
shape of the record that would be populated by the reader (i.e. contains
@@ -51,10 +65,10 @@ def create_template_row(schema: type[BaseModel]) -> dict[str, Any]:
"""
template_row: dict[str, Any] = {}
- for field_name, model_field_def in schema.__fields__.items():
- field_type = _strip_annotated(model_field_def.annotation)
+ for field_name, model_field_def in schema.model_fields.items(): # type: ignore
+ field_type = _strip_optional(_strip_annotated(model_field_def.annotation))
- if not model_field_def.is_complex():
+ if not is_type_complex(field_type):
template_row[field_name] = None
continue
@@ -70,8 +84,8 @@ def create_template_row(schema: type[BaseModel]) -> dict[str, Any]:
# This is a quick and dirty hack to avoid implementing our own logic
# to check complex types...
- list_type_field_spec = create_model("", lt=(list_type, ...)).__fields__["lt"]
- if not list_type_field_spec.is_complex():
+ list_type_field_spec = create_model("", lt=(list_type, ...)).model_fields["lt"]
+ if not is_field_complex(list_type_field_spec):
template_row[field_name] = [None]
continue
@@ -329,8 +343,8 @@ def write_parquet( # type: ignore
target_location = file_uri_to_local_path(target_location).as_posix()
if schema:
polars_schema: dict[str, pl.DataType] = { # type: ignore
- fld.name: get_polars_type_from_annotation(fld.type_)
- for fld in stringify_model(schema).__fields__.values()
+ name: get_polars_type_from_annotation(fld.type_)
+ for name, fld in stringify_model(schema).model_fields.items()
}
polars_schema[RECORD_INDEX_COLUMN_NAME] = get_polars_type_from_annotation(int)
pl.LazyFrame(data=entity, schema=polars_schema).sink_parquet(
diff --git a/src/dve/core_engine/backends/utilities.py b/src/dve/core_engine/backends/utilities.py
index ee4c2ab..6b8c5bc 100644
--- a/src/dve/core_engine/backends/utilities.py
+++ b/src/dve/core_engine/backends/utilities.py
@@ -4,11 +4,12 @@
from dataclasses import is_dataclass
from datetime import date, datetime, time
from decimal import Decimal
-from typing import Any, ClassVar, GenericAlias, Union # type: ignore
+from typing import Any, ClassVar, GenericAlias, Literal, Union # type: ignore
import polars as pl # type: ignore
from polars.datatypes.classes import DataTypeClass as PolarsType
from pydantic import BaseModel, create_model
+from pydantic.fields import FieldInfo
from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type
from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
@@ -38,6 +39,26 @@
"""A mapping of Python types to the equivalent Polars types."""
+def is_type_complex(type_: Any) -> bool:
+ """Check whether a type is a complex type or not."""
+ if type_ in (str, int, float, bool, bytes):
+ return False
+
+ if type_ in (date, datetime, time, Decimal):
+ return False
+
+ return True
+
+
+def is_field_complex(field: FieldInfo) -> bool:
+ """
+ Replacement function for Pydantic v1 `is_complex` check provided
+ by the v1 ModelField object.
+ """
+ type_annotation = field.annotation
+ return is_type_complex(type_annotation)
+
+
def stringify_type(type_: Union[type, GenericAlias]) -> type:
"""Stringify an individual type."""
if isinstance(type_, type) and not isinstance(
@@ -46,7 +67,7 @@ def stringify_type(type_: Union[type, GenericAlias]) -> type:
if issubclass(type_, BaseModel):
return stringify_model(type_)
- is_complex = create_model("", t=(type_, ...)).__fields__["t"].is_complex()
+ is_complex = is_field_complex(create_model("", t=(type_, ...)).model_fields["t"])
if not is_complex: # A non-container type, return string.
return str
@@ -68,9 +89,9 @@ def stringify_type(type_: Union[type, GenericAlias]) -> type:
def stringify_model(model: type[BaseModel]) -> type[BaseModel]:
"""Stringify a `pydantic` model."""
fields = {}
- for field_name, field in model.__fields__.items():
- fields[field_name] = (stringify_type(field.annotation), ...)
- return create_model(model.__name__, **fields) # type: ignore
+ for field_name, field in model.model_fields.items():
+ fields[field_name] = (stringify_type(field.annotation), ...) # type: ignore
+ return create_model(model.__class__.__name__, **fields) # type: ignore
def dedup_messages(messages: Messages) -> Messages:
@@ -106,7 +127,7 @@ def get_polars_type_from_annotation(type_annotation: Any) -> PolarsType:
'optional' wrapper and return the inner type
- A subclass of `typing.TypedDict` with values typed using supported types. This
will parse the value types as Polars types and return a Polars Struct.
- - A dataclass or `pydantic.main.ModelMetaClass` with values typed using supported types.
+ - A dataclass or `pydantic.BaseModel` with values typed using supported types.
This will parse the field types as Polars types and return a Polars Struct.
- Any supported type, with a `typing_extensions.Annotated` wrapper.
- A `decimal.Decimal` wrapped with `typing_extensions.Annotated` with a `DecimalConfig`
@@ -120,6 +141,15 @@ def get_polars_type_from_annotation(type_annotation: Any) -> PolarsType:
"""
type_origin = get_origin(type_annotation)
+ if type_origin is Literal:
+ # TODO - look at using _get_non_heterogenous_type instead?
+ polars_types = [get_polars_type_from_annotation(type(t)) for t in get_args(type_annotation)]
+ if not polars_types or not all(t == polars_types[0] for t in polars_types):
+ raise ValueError(
+ f"Unable to determine a single concrete type for Literal. Got {type_annotation!r}"
+ )
+ return polars_types[0]
+
# An `Optional` or `Union` type, check to ensure non-heterogenity.
if type_origin is Union:
python_type = _get_non_heterogenous_type(get_args(type_annotation))
diff --git a/src/dve/core_engine/configuration/v1/__init__.py b/src/dve/core_engine/configuration/v1/__init__.py
index 04d5e07..959596f 100644
--- a/src/dve/core_engine/configuration/v1/__init__.py
+++ b/src/dve/core_engine/configuration/v1/__init__.py
@@ -3,8 +3,8 @@
import json
from typing import Any, Optional, Union
-from pydantic import BaseModel, Field, PrivateAttr, validate_arguments
-from typing_extensions import Annotated, Literal
+from pydantic import BaseModel, Field, PrivateAttr, validate_call
+from typing_extensions import Literal
from dve.core_engine.backends.base.reference_data import ReferenceConfig, ReferenceConfigUnion
from dve.core_engine.backends.metadata.contract import DataContractMetadata, ReaderConfig
@@ -178,7 +178,7 @@ class V1EngineConfig(BaseEngineConfig):
)
"""Rule store rules from the loaded rule stores."""
- @validate_arguments
+ @validate_call
def _update_rule_store(self, rule_store: dict[RuleName, BusinessComponentSpecConfigUnion]):
"""Update the rule store rules to add/override the rules from the new store."""
self._rule_store_rules.update(rule_store) # pylint: disable=E1101
@@ -281,7 +281,9 @@ def _load_rules_and_vars(self) -> tuple[list[Rule], list[TemplateVariables]]:
rules, local_variable_list = [], []
added_rules: set[RuleName] = set()
- for index, complex_rule_config in enumerate(self.transformations.complex_rules): # pylint: disable=E1101
+ for index, complex_rule_config in enumerate(
+ self.transformations.complex_rules # pylint: disable=E1101
+ ):
rule, local_params, deps = self._resolve_business_rule(complex_rule_config)
missing_rules = deps - added_rules
if missing_rules:
@@ -309,7 +311,7 @@ def get_contract_metadata(self) -> DataContractMetadata:
validators = {}
reporting_fields = {}
- contract_dict = self.contract.dict()
+ contract_dict = self.contract.model_dump()
error_info = {}
if self.contract.error_details:
error_info = self.load_error_message_info(self.contract.error_details)
diff --git a/src/dve/core_engine/configuration/v1/steps.py b/src/dve/core_engine/configuration/v1/steps.py
index f795c8c..6b30a4a 100644
--- a/src/dve/core_engine/configuration/v1/steps.py
+++ b/src/dve/core_engine/configuration/v1/steps.py
@@ -10,9 +10,9 @@
# pylint: disable=missing-class-docstring
from abc import ABC, abstractmethod
-from typing import Any, Optional, Union
+from typing import Optional, Union
-from pydantic import BaseModel, Extra, Field, validator
+from pydantic import BaseModel, Field, ValidationInfo, field_validator
from typing_extensions import Annotated, Literal
from dve.core_engine.backends.metadata.rules import (
@@ -40,10 +40,9 @@
class ConfigStep(BaseModel, ABC):
"""The parent for the config steps."""
- class Config: # pylint: disable=too-few-public-methods
- """Config class for dynamically generated pydantic models"""
-
- extra = Extra.forbid
+ model_config = {
+ "extra": "forbid",
+ }
name: Optional[str] = None
"""The 'name' of the rule. This is mapped to an ID in the entity."""
@@ -107,10 +106,10 @@ class GroupByConfig(ConfigStep):
pivot_values: Optional[list[str]] = None
agg_columns: MultipleExpressions
- @validator("pivot_values")
+ @field_validator("pivot_values")
@classmethod
- def _ensure_no_values_if_not_column(cls, value: Optional[str], values: dict[str, Any]):
- if value and not values["pivot_column"]:
+ def _ensure_no_values_if_not_column(cls, value: Optional[str], info: ValidationInfo):
+ if value and not info.data.get("pivot_column"):
raise ValueError("Cannot provide 'pivot_values' if no 'pivot_column'")
return value
diff --git a/src/dve/core_engine/engine.py b/src/dve/core_engine/engine.py
index b2ec9a7..a7931ad 100644
--- a/src/dve/core_engine/engine.py
+++ b/src/dve/core_engine/engine.py
@@ -4,9 +4,17 @@
import logging
from pathlib import Path
from types import TracebackType
-from typing import Any, Optional, Union
-
-from pydantic import BaseModel, Field, PrivateAttr, validate_arguments, validator
+from typing import Annotated, Optional, Union
+
+from pydantic import (
+ BaseModel,
+ ConfigDict,
+ Field,
+ PrivateAttr,
+ ValidationInfo,
+ field_validator,
+ validate_call,
+)
from pydantic.types import FilePath
from pyspark.sql import SparkSession
@@ -26,11 +34,7 @@
class CoreEngine(BaseModel):
"""The core engine implementation for the data validation engine."""
- class Config: # pylint: disable=too-few-public-methods
- """`pydantic` configuration options."""
-
- arbitrary_types_allowed = True
- validate_assignment = True
+ model_config = {"arbitrary_types_allowed": True, "validate_assignment": True}
backend_config: BaseEngineConfig
"""The backend configuration for the given run."""
@@ -52,16 +56,20 @@ class Config: # pylint: disable=too-few-public-methods
Data will be chunked to parquet in this directory after being read,
and written here before filters are applied.
-
"""
- backend: BaseBackend = None # type: ignore
+ # TODO - recommended to not use validate_default like this, but for now will use as replacement to always=True # pylint: disable=C0301
+ # see https://pydantic.dev/docs/validation/latest/get-started/migration/#validator-and-root_validator-are-deprecated # pylint: disable=C0301
+ backend: Annotated[Optional[BaseBackend], Field(default=None, validate_default=True)]
"""The backend to use to process the files."""
+
debug: bool = False
"""Indication of if this run is in debug mode."""
- @validator("cache_prefix_uri", "output_prefix_uri", allow_reuse=True, pre=True)
+ @field_validator("cache_prefix_uri", "output_prefix_uri", mode="before")
# pylint: disable=E0213
- def _validate_prefix_uri(cls, location: Optional[Location]) -> Optional[URI]:
+ def _validate_prefix_uri(
+ cls, location: Optional[Location], info: ValidationInfo # pylint: disable=W0613
+ ) -> Optional[URI]:
"""Ensure we support the cache prefix scheme."""
if location is None:
return None
@@ -71,25 +79,25 @@ def __init__(self, *args, **kwargs):
# pylint: disable=W0235
super().__init__(*args, **kwargs)
- @validator("backend", always=True)
+ @field_validator("backend")
@classmethod
- def _ensure_backend(cls, value: Optional[BaseBackend], values: dict[str, Any]) -> BaseBackend:
+ def _ensure_backend(cls, value: Optional[BaseBackend], info: ValidationInfo) -> BaseBackend:
"""Ensure a default backend is created if a backend is not specified."""
if value is not None:
return value
- main_logger = values.get("main_log")
+ main_logger = info.data.get("main_log")
if main_logger is None:
- return SparkBackend(dataset_config_uri=values.get("dataset_config_uri"))
+ return SparkBackend(dataset_config_uri=info.data.get("dataset_config_uri"))
return SparkBackend(
- dataset_config_uri=values.get("dataset_config_uri"),
+ dataset_config_uri=info.data.get("dataset_config_uri"),
logger=get_child_logger(
".".join((SparkBackend.__module__, SparkBackend.__name__)), main_logger
),
)
@classmethod
- @validate_arguments(config={"arbitrary_types_allowed": True})
+ @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
def build(
cls,
dataset_config_path: Union[FilePath, URI],
@@ -152,7 +160,7 @@ def build_from_model(cls, model_str: JSONstring):
"""
main_log = get_logger("CoreEngine")
main_log.info("Initalise from model...")
- return cls.build(**EngineRunValidation(**json.loads(model_str)).dict())
+ return cls.build(**EngineRunValidation(**json.loads(model_str)).model_dump())
def __enter__(self) -> "CoreEngine":
self.main_log.info("Entering pipeline context.") # pylint: disable=E1101
@@ -170,7 +178,9 @@ def __exit__(
exc_value: Optional[Exception],
traceback: Optional[TracebackType],
) -> None:
- self.main_log.info(f"Exiting pipeline context, clearing {self.cache_prefix!r}") # pylint: disable=E1101
+ self.main_log.info( # pylint: disable=E1101
+ f"Exiting pipeline context, clearing {self.cache_prefix!r}"
+ )
cache_dir = self._cache_dir
self._cache_dir = None
@@ -198,7 +208,9 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities:
"""
output_entities = {}
- self.main_log.info(f"Writing entities to the output location: {self.output_prefix_uri}") # pylint: disable=E1101
+ self.main_log.info( # pylint: disable=E1101
+ f"Writing entities to the output location: {self.output_prefix_uri}"
+ )
for entity_name, entity in entities.items():
entity = entity.drop(RECORD_INDEX_COLUMN_NAME)
@@ -206,9 +218,13 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities:
output_uri = joinuri(self.output_prefix_uri, entity_name)
if get_resource_exists(output_uri):
- self.main_log.info(f"{output_uri} already exists - will be overwritten") # pylint: disable=E1101
+ self.main_log.info( # pylint: disable=E1101
+ f"{output_uri} already exists - will be overwritten"
+ )
- self.main_log.info(f"+ Writing parquet output to {output_uri!r}") # pylint: disable=E1101
+ self.main_log.info( # pylint: disable=E1101
+ f"+ Writing parquet output to {output_uri!r}"
+ )
entity.write.mode("overwrite").parquet(output_uri)
spark_session = SparkSession.builder.getOrCreate()
output_entities[entity_name] = spark_session.read.format("parquet").load(
@@ -255,11 +271,16 @@ def run_pipeline(
references should be valid after the pipeline context exits.
"""
- entities, errors_uri = self.backend.process_legacy(
- self.output_prefix_uri,
- entity_locations,
- self.backend_config.get_contract_metadata(),
- self.backend_config.get_rule_metadata(),
- submission_info,
+ if self.backend:
+ entities, errors_uri = self.backend.process_legacy(
+ self.output_prefix_uri,
+ entity_locations,
+ self.backend_config.get_contract_metadata(),
+ self.backend_config.get_rule_metadata(),
+ submission_info,
+ )
+ return self._write_outputs(entities), errors_uri
+
+ raise AttributeError(
+ "Backend implementation not defined. Cannot run the pipeline without a defined backend. Choose DuckDB or Spark" # pylint: disable=C0301
)
- return self._write_outputs(entities), errors_uri
diff --git a/src/dve/core_engine/loggers.py b/src/dve/core_engine/loggers.py
index 036c3d2..6c9b3aa 100644
--- a/src/dve/core_engine/loggers.py
+++ b/src/dve/core_engine/loggers.py
@@ -18,7 +18,7 @@ def filter(self, record):
class UTCFormatter(logging.Formatter): # pragma: no cover
"""A formatter with timestamps in the UTC timezone."""
- converter = time.gmtime
+ converter = time.gmtime # type: ignore
def get_default_handler() -> logging.Handler: # pragma: no cover
diff --git a/src/dve/core_engine/message.py b/src/dve/core_engine/message.py
index 627ae3a..2980990 100644
--- a/src/dve/core_engine/message.py
+++ b/src/dve/core_engine/message.py
@@ -10,7 +10,7 @@
from functools import reduce
from typing import Any, ClassVar, Optional, Union
-from pydantic import BaseModel, ValidationError, validator
+from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, field_validator
from pydantic.dataclasses import dataclass
from dve.core_engine.constants import CONTRACT_ERROR_VALUE_FIELD_NAME, RECORD_INDEX_COLUMN_NAME
@@ -83,12 +83,6 @@ def extract_error_value(records, error_location):
"""
-class Config: # pylint: disable=too-few-public-methods
- """`pydantic` configuration options."""
-
- arbitrary_types_allowed = True
-
-
# pylint: disable=R0902
@dataclass
class UserMessage:
@@ -130,7 +124,7 @@ def is_critical(self) -> bool:
return self.FailureType == "integrity"
-@dataclass(config=Config, eq=True)
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True), eq=True)
class FeedbackMessage: # pylint: disable=too-many-instance-attributes
"""Information which affects processing and needs to be feeded back."""
@@ -195,9 +189,11 @@ class FeedbackMessage: # pylint: disable=too-many-instance-attributes
]
"""The header that should be written to CSV."""
- @validator("reporting_field")
+ @field_validator("reporting_field")
# pylint: disable=no-self-argument
- def _split_reporting_field(cls, value) -> Union[list[str], str, None]:
+ def _split_reporting_field(
+ cls, value: Optional[str | list[str]], info: ValidationInfo # pylint: disable=W0613
+ ) -> Union[list[str], str, None]:
if isinstance(value, list):
return value
if isinstance(value, str):
@@ -210,9 +206,11 @@ def _split_reporting_field(cls, value) -> Union[list[str], str, None]:
return value
return None
- @validator("error_location", pre=True)
+ @field_validator("error_location", mode="before")
# pylint: disable=no-self-argument
- def _validate_error_location(cls, value: Any) -> Optional[str]:
+ def _validate_error_location(
+ cls, value: Optional[str], info: ValidationInfo # pylint: disable=W0613
+ ) -> Optional[str]:
"""Format error location to a string."""
if value is None:
return None # pragma: no cover
@@ -247,7 +245,8 @@ def from_pydantic_error(
messages: Messages = []
for error_dict in error.errors():
error_type = error_dict["type"]
- if "none.not_allowed" in error_type or "value_error.missing" in error_type:
+ _input = error_dict["input"]
+ if "missing" in error_type or _input is None:
category = "Blank"
else:
category = "Bad value"
@@ -267,6 +266,8 @@ def from_pydantic_error(
is_informational = False
if error_code.endswith("warning"):
is_informational = True
+ # TODO - this should copy the default error detail and then update with any custom codes defined in error_details pylint: disable=C0301
+ # TODO - to ensure that user does not need to define all custom details (i.e. bad value custom, blank default) pylint: disable=C0301
error_detail: DataContractErrorDetail = error_details.get( # type: ignore
error_field, DEFAULT_ERROR_DETAIL
).get(category)
diff --git a/src/dve/core_engine/models.py b/src/dve/core_engine/models.py
index 09fcbb3..1d84bab 100644
--- a/src/dve/core_engine/models.py
+++ b/src/dve/core_engine/models.py
@@ -11,7 +11,15 @@
from pathlib import Path
from typing import Any, Optional
-from pydantic import UUID4, BaseModel, Field, FilePath, root_validator, validator
+from pydantic import (
+ UUID4,
+ BaseModel,
+ Field,
+ FilePath,
+ ValidationInfo,
+ field_validator,
+ model_validator,
+)
from dve.core_engine.backends.metadata.contract import ReaderConfig
from dve.core_engine.type_hints import EntityName, ProcessingStatus, SubmissionResult
@@ -26,16 +34,17 @@ class AuditRecord(BaseModel):
submission_id: str
"""Unique id of the submission"""
+ # todo - why bother supplying a date_updated here when it gets overwritten by time? Only works if you supply both values # pylint: disable=C0301
date_updated: Optional[dt.date] = None
"""The date the record was added to the table"""
time_updated: Optional[dt.datetime] = Field(default_factory=dt.datetime.now)
"""The timestamp the record was added to the table"""
- @root_validator(allow_reuse=True)
- def populate_date_updated(cls, values): # pylint: disable=no-self-argument
+ @model_validator(mode="after")
+ def populate_date_updated(self):
"""Add date_updated from time_updated value"""
- values["date_updated"] = values["time_updated"].date()
- return values
+ self.date_updated = self.time_updated.date() # pylint: disable=E1101
+ return self
class SubmissionInfoMismatchWarning(UserWarning):
@@ -64,8 +73,10 @@ class SubmissionInfo(AuditRecord):
datetime_received: Optional[dt.datetime] = None # type: ignore
"""The datetime the file was received."""
- @validator("file_extension")
- def _ensure_just_file_stem(cls, extension: str): # pylint: disable=no-self-argument
+ @field_validator("file_extension")
+ def _ensure_just_file_stem(
+ cls, extension: str, info: ValidationInfo # pylint: disable=W0613
+ ): # pylint: disable=no-self-argument
return extension.rsplit(".", 1)[-1]
@property
@@ -95,8 +106,8 @@ def __eq__(self, other: object) -> bool:
if not isinstance(other, SubmissionInfo):
raise NotImplementedError("Unable to determine equality if not a SubmissionInfo object")
_exclude = ["date_updated", "time_updated"]
- return {k: v for k, v in self.dict().items() if k not in _exclude} == {
- k: v for k, v in other.dict().items() if k not in _exclude
+ return {k: v for k, v in self.model_dump().items() if k not in _exclude} == {
+ k: v for k, v in other.model_dump().items() if k not in _exclude
}
@@ -116,8 +127,8 @@ def __eq__(self, other: object) -> bool:
"Unable to determine equality if not a SubmissionStatisticsRecord object"
) # pylint: disable=line-too-long
_exclude = ["date_updated", "time_updated"]
- return {k: v for k, v in self.dict().items() if k not in _exclude} == {
- k: v for k, v in other.dict().items() if k not in _exclude
+ return {k: v for k, v in self.model_dump().items() if k not in _exclude} == {
+ k: v for k, v in other.model_dump().items() if k not in _exclude
}
@@ -139,9 +150,9 @@ class ProcessingStatusRecord(AuditRecord):
processing_status: ProcessingStatus
"""The processing status of the submission"""
- job_run_id: Optional[int]
+ job_run_id: Optional[int] = None
"""The run id of the databricks job used to process the submission"""
- submission_result: Optional[SubmissionResult]
+ submission_result: Optional[SubmissionResult] = None
"""Whether the file validation was a success or failure"""
@@ -156,9 +167,9 @@ class EngineRun(BaseModel):
# TODO: What if we want to set an alt/override output prefix
# and not have the submission_id appended to it?
- @validator("output_prefix")
- def _set_output_path(cls, prefix, values: dict): # pylint: disable=E0213
- v_id = values.get("submission_id")
+ @field_validator("output_prefix")
+ def _set_output_path(cls, prefix, info: ValidationInfo): # pylint: disable=E0213
+ v_id = info.data.get("submission_id")
if v_id:
return os.path.join(prefix, str(v_id))
return prefix
@@ -181,8 +192,11 @@ class ConcreteEntity(EntitySpecification, arbitrary_types_allowed=True):
"""An optional key field to use for the entity."""
reporting_fields: Optional[list[str]] = None
- @validator("reporting_fields", pre=True)
- def _ensure_list(cls, value: Optional[str]) -> Optional[list[str]]: # pylint: disable=E0213
+ @field_validator("reporting_fields", mode="before")
+ @classmethod
+ def _ensure_list(
+ cls, value: Optional[str], info: ValidationInfo # pylint: disable=W0613
+ ) -> Optional[list[str]]:
"""Ensure the reporting fields are a list."""
if value is None:
return None
diff --git a/src/dve/core_engine/validation.py b/src/dve/core_engine/validation.py
index f62309b..05f2e2b 100644
--- a/src/dve/core_engine/validation.py
+++ b/src/dve/core_engine/validation.py
@@ -6,8 +6,7 @@
from typing import Optional
from pyarrow.lib import RecordBatch # type: ignore
-from pydantic import ValidationError
-from pydantic.main import ModelMetaclass
+from pydantic import BaseModel, ValidationError
from dve.core_engine.message import DEFAULT_ERROR_DETAIL, DataContractErrorDetail, FeedbackMessage
from dve.core_engine.type_hints import ContractContents, EntityName, ErrorCategory, Messages, Record
@@ -36,7 +35,7 @@ def __init__(
self._model_definition = model_definition
self._validators = validators
self.entity_name = entity_name
- self._model: Optional[ModelMetaclass] = None
+ self._model: Optional[BaseModel] = None
self._error_info = error_info or {}
self._error_details: Optional[
dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]
@@ -48,7 +47,7 @@ def __reduce__(self): # Don't attempt to pickle Pydantic models.
return super().__reduce__()
@property
- def model(self) -> ModelMetaclass:
+ def model(self) -> BaseModel:
"""The loaded pydantic model for the entity."""
if not self._model:
models = JSONtoPyd(self._model_definition).generate_models(
@@ -83,7 +82,7 @@ def __call__(self, record: Record) -> tuple[Optional[Record], Messages]:
messages: Messages = []
try:
# pylint: disable=not-callable
- validated: Record = self.model(**record).dict()
+ validated: Record = self.model(**record).model_dump() # type: ignore
except ValidationError as err:
# we still want to report warnings
# when a record is invalid
diff --git a/src/dve/metadata_parser/domain_types.py b/src/dve/metadata_parser/domain_types.py
index 84dc5a7..3d7bc3c 100644
--- a/src/dve/metadata_parser/domain_types.py
+++ b/src/dve/metadata_parser/domain_types.py
@@ -1,15 +1,18 @@
"""Domain specific type definitions for use in validators."""
# pylint: disable=too-few-public-methods
+# pylint: disable=W0613
+
import datetime as dt
import itertools
import re
import warnings
-from collections.abc import Iterator, Sequence
+from collections.abc import Sequence
from functools import lru_cache
-from typing import ClassVar, Optional, TypeVar, Union
+from typing import Any, ClassVar, Optional, TypeVar, Union
-from pydantic import fields, types, validate_arguments
+from pydantic import GetCoreSchemaHandler, types, validate_call
+from pydantic_core import CoreSchema, core_schema
from typing_extensions import Literal
from dve.metadata_parser import exc
@@ -32,16 +35,16 @@
POSTCODE_REGEX = re.compile(r"\A[a-zA-Z]{1,2}\d([a-zA-Z]?|\d?)\s\d[a-zA-Z]{2}\Z")
-class _SimpleRegexValidator(types.ConstrainedStr):
+class _SimpleRegexValidator:
"""A basic regex-validated type."""
- regex: re.Pattern
+ pattern: re.Pattern
"""A regex pattern used to validate the string."""
strip_whitespace: bool = True
"""Whether to strip the whitespace from the string."""
-class NHSNumber(types.ConstrainedStr):
+class NHSNumber(str):
"""A constrained string which validates an NHS number.
### Validation criteria
@@ -98,7 +101,7 @@ class NHSNumber(types.ConstrainedStr):
warn_on_test_numbers = True
@classmethod
- def _warn_for_possible_invalid_number(cls, nhs_number: str, loc: str) -> None:
+ def _warn_for_possible_invalid_number(cls, nhs_number: str) -> None:
"""Emit warnings for possible invalid NHS numbers."""
reason = None
@@ -111,10 +114,10 @@ def _warn_for_possible_invalid_number(cls, nhs_number: str, loc: str) -> None:
reason = "NHS number is a palindrome: this indicates a test number"
if reason:
- warnings.warn(exc.LocWarning(f"NHS number possibly invalid ({reason})", loc))
+ warnings.warn(exc.LocWarning(f"NHS number possibly invalid ({reason})"))
@staticmethod
- def ensure_format(nhs_number: Optional[str]) -> str:
+ def ensure_format(nhs_number: Optional[str | int]) -> str:
"""Coerce an NHS number string to the correct format, raising an error if
coersion fails.
@@ -139,7 +142,7 @@ def confirm_checksum_validates(nhs_number: str) -> bool:
return check == int(check_digit)
@classmethod
- def check_validates(cls, value: Optional[str]) -> bool:
+ def check_validates(cls, value: str) -> bool:
"""Check whether an NHS number is valid, returning `True` for valid numbers
and `False` for invalid numbers.
@@ -152,19 +155,24 @@ def check_validates(cls, value: Optional[str]) -> bool:
return is_valid
@classmethod
- def validate(cls, value: Optional[str], field: fields.ModelField) -> str: # type: ignore # pylint: disable=W0221
+ def validate(cls, value: Optional[str | int]) -> str: # type: ignore # pylint: disable=W0221
"""Validates the given postcode"""
nhs_number = cls.ensure_format(value)
if cls.confirm_checksum_validates(nhs_number):
- # TODO: Get a better way to get 'loc' here.
- cls._warn_for_possible_invalid_number(nhs_number, field.name)
+ cls._warn_for_possible_invalid_number(nhs_number)
return nhs_number
raise ValueError("NHS number invalid (incorrect check digit: cannot be a real NHS number)")
+ @classmethod
+ def __get_pydantic_core_schema__(
+ cls, source_type: Any, handler: GetCoreSchemaHandler
+ ) -> CoreSchema:
+ return core_schema.no_info_plain_validator_function(cls.validate)
+
@lru_cache()
-@validate_arguments
+@validate_call
def permissive_nhs_number(warn_on_test_numbers: bool = False):
"""Defaults to not checking for test numbers"""
dict_ = NHSNumber.__dict__.copy()
@@ -173,10 +181,10 @@ def permissive_nhs_number(warn_on_test_numbers: bool = False):
return type("NHSNumber", (NHSNumber, *NHSNumber.__bases__), dict_)
-class Postcode(types.ConstrainedStr):
+class Postcode(str):
"""Postcode constrained string"""
- regex: re.Pattern = POSTCODE_REGEX
+ pattern: re.Pattern = POSTCODE_REGEX
strip_whitespace = True
apply_normalize = True
@@ -198,14 +206,25 @@ def validate(cls, value: str) -> Optional[str]: # type: ignore
if not value:
return None
- if not cls.regex.match(value):
+ if not cls.pattern.match(value):
raise ValueError("Invalid Postcode submitted")
return value
+ @classmethod
+ def __get_pydantic_core_schema__(
+ cls, source_type: Any, handler: GetCoreSchemaHandler
+ ) -> CoreSchema:
+ return core_schema.chain_schema(
+ [
+ handler(str),
+ core_schema.no_info_plain_validator_function(cls.validate),
+ ]
+ )
+
@lru_cache()
-@validate_arguments
+@validate_call
def postcode(
# pylint: disable=R0913
strip_whitespace: Optional[bool] = True,
@@ -214,8 +233,7 @@ def postcode(
strict: Optional[bool] = False,
min_length: Optional[int] = None,
max_length: Optional[int] = None,
- curtail_length: Optional[int] = None,
- regex: Optional[str] = POSTCODE_REGEX, # type: ignore
+ pattern: Optional[str] = POSTCODE_REGEX, # type: ignore
apply_normalize: Optional[bool] = True,
) -> type[Postcode]:
"""Return a formatted date class with a set date format
@@ -229,8 +247,7 @@ def postcode(
dict_["strict"] = strict
dict_["min_length"] = min_length
dict_["max_length"] = max_length
- dict_["curtail_length"] = curtail_length
- dict_["regex"] = regex
+ dict_["pattern"] = pattern
dict_["apply_normalize"] = apply_normalize
return type("Postcode", (Postcode, *Postcode.__bases__), dict_)
@@ -244,17 +261,17 @@ class OrgID(_SimpleRegexValidator):
"""
- regex = re.compile(r"^[A-Z0-9]{3,5}$")
+ pattern = re.compile(r"^[A-Z0-9]{3,5}$")
strip_whitespace = False
@classmethod
- def validate(cls, value: str) -> str:
- """Validates the given OrgID"""
- if not value:
- raise ValueError("org_id not provided")
- return super().validate(value)
+ def __get_pydantic_core_schema__(
+ cls, source_type: Any, handler: GetCoreSchemaHandler
+ ) -> CoreSchema:
+ return core_schema.str_schema(pattern=cls.pattern, strip_whitespace=cls.strip_whitespace)
+# TODO - look into replacing datetime types with AwareDatetime and NaiveDatetime from pydantic v2
class ConFormattedDate(dt.date):
"""A date, provided as a date or a string in a specific format."""
@@ -274,12 +291,9 @@ class ConFormattedDate(dt.date):
@classmethod
def validate(cls, value: Optional[Union[dt.date, str]]) -> Optional[dt.date]:
"""Validate a passed datetime or string."""
- if value is None:
- return value
-
if isinstance(value, dt.date):
date = value
- elif cls.DATE_FORMAT is not None:
+ elif cls.DATE_FORMAT is not None and value:
try:
date = dt.datetime.strptime(value, cls.DATE_FORMAT).date()
if cls.strict and (date.strftime(cls.DATE_FORMAT) != value):
@@ -311,14 +325,20 @@ def validate_range(cls, value) -> Optional[dt.date]:
return value
@classmethod
- def __get_validators__(cls) -> Iterator[classmethod]:
- """Gets all validators"""
- yield cls.validate # type: ignore
- yield cls.validate_range # type: ignore
+ def __get_pydantic_core_schema__(
+ cls, source_type: Any, handler: GetCoreSchemaHandler
+ ) -> CoreSchema:
+ return core_schema.chain_schema(
+ [
+ handler(str),
+ core_schema.no_info_plain_validator_function(cls.validate),
+ core_schema.no_info_plain_validator_function(cls.validate_range),
+ ]
+ )
@lru_cache()
-@validate_arguments
+@validate_call
def conformatteddate(
date_format: Optional[str] = None,
strict: Optional[bool] = False,
@@ -381,6 +401,7 @@ def reformat_nhs_string_format(string: str) -> str:
)
)
+ # TODO - check this hasn't broken as pydantic.parse_datetime retired although this not using pydantic datetime AFAIK pylint: disable=C0301
@classmethod
def parse_datetime(cls, string: str) -> dt.datetime:
"""Attempt to parse a datetime using various formats in sequence."""
@@ -403,6 +424,8 @@ def parse_datetime(cls, string: str) -> dt.datetime:
@classmethod
def validate(cls, value: Optional[Union[dt.datetime, str]]) -> Optional[dt.datetime]:
"""Validate a passed datetime or string."""
+ # TODO - This check is simply needed because of the test in test_domain_types which is possibly invalid post pylint: disable=C0301
+ # Pydantic v2 upgrade now as NoneType will be handled by the handler
if value is None:
return value
@@ -427,9 +450,17 @@ def validate(cls, value: Optional[Union[dt.datetime, str]]) -> Optional[dt.datet
return datetime
@classmethod
- def __get_validators__(cls) -> Iterator[classmethod]:
+ def __get_pydantic_core_schema__(
+ cls, source_type: Any, handler: GetCoreSchemaHandler
+ ) -> CoreSchema:
"""Gets all validators"""
- yield cls.validate # type: ignore
+ # yield cls.validate # type: ignore
+ return core_schema.chain_schema(
+ [
+ handler(str),
+ core_schema.no_info_plain_validator_function(cls.validate),
+ ]
+ )
class FormattedTime(dt.time):
@@ -496,9 +527,6 @@ def parse_time(cls, string: str) -> dt.time:
@classmethod
def validate(cls, value: Union[dt.time, dt.datetime, str]) -> dt.time | None:
"""Validate a passed time, datetime or string."""
- if value is None:
- return value
-
if isinstance(value, dt.time):
new_time = value
elif isinstance(value, dt.datetime):
@@ -524,13 +552,20 @@ def validate(cls, value: Union[dt.time, dt.datetime, str]) -> dt.time | None:
return new_time
@classmethod
- def __get_validators__(cls) -> Iterator[classmethod]:
+ def __get_pydantic_core_schema__(
+ cls, source_type: Any, handler: GetCoreSchemaHandler
+ ) -> CoreSchema:
"""Gets all validators"""
- yield cls.validate # type: ignore
+ return core_schema.chain_schema(
+ [
+ handler(str),
+ core_schema.no_info_plain_validator_function(cls.validate),
+ ]
+ )
@lru_cache()
-@validate_arguments
+@validate_call
def formatteddatetime(
date_format: Optional[str] = None,
timezone_treatment: Literal["forbid", "permit", "require"] = "permit",
@@ -550,7 +585,7 @@ def formatteddatetime(
@lru_cache()
-@validate_arguments
+@validate_call
def formattedtime(
time_format: Optional[str] = None,
timezone_treatment: Literal["forbid", "permit", "require"] = "permit",
@@ -612,13 +647,20 @@ def validate(cls, value: Union[dt.date, str]) -> Optional[dt.date]:
return value
@classmethod
- def __get_validators__(cls) -> Iterator[classmethod]:
+ def __get_pydantic_core_schema__(
+ cls, source_type: Any, handler: GetCoreSchemaHandler
+ ) -> CoreSchema:
"""Gets all validators"""
- yield cls.validate # type: ignore
+ return core_schema.chain_schema(
+ [
+ handler(dt.date),
+ core_schema.no_info_plain_validator_function(cls.validate),
+ ]
+ )
# @lru_cache()
-@validate_arguments
+@validate_call
def reportingperiod(
reporting_period_type: Literal["start", "end"], date_format: Optional[str] = "%Y-%m-%d"
) -> type[ReportingPeriod]:
@@ -632,11 +674,12 @@ def reportingperiod(
return type("ReportingPeriod", (ReportingPeriod, *ReportingPeriod.__bases__), dict_)
+# TODO - refactor this as it won't work in Pyndatic V2
@lru_cache()
-@validate_arguments
+@validate_call
def alphanumeric(
- min_digits: types.NonNegativeInt = 1,
- max_digits: types.PositiveInt = 1,
+ min_digits: types.NonNegativeInt = 1, # pylint: disable=E1101
+ max_digits: types.PositiveInt = 1, # pylint: disable=E1101
) -> type[_SimpleRegexValidator]:
"""Return a regex-validated class which will ensure that
passed numbers are alphanumeric.
@@ -651,7 +694,7 @@ def alphanumeric(
pattern_str = f"{an_group_str}{{{min_digits},{max_digits}}}"
dict_ = _SimpleRegexValidator.__dict__.copy()
- dict_["regex"] = re.compile(f"^{pattern_str}$")
+ dict_["pattern"] = re.compile(f"^{pattern_str}$")
return type(
type_name,
@@ -660,11 +703,12 @@ def alphanumeric(
)
+# TODO - refactor this as it won't work in Pyndatic V2
@lru_cache()
-@validate_arguments
+@validate_call
def identifier(
- min_digits: types.NonNegativeInt = 1,
- max_digits: types.PositiveInt = 1,
+ min_digits: types.NonNegativeInt = 1, # pylint: disable=E1101
+ max_digits: types.PositiveInt = 1, # pylint: disable=E1101
) -> type[_SimpleRegexValidator]:
"""
Return a regex-validated class which will ensure that
@@ -680,7 +724,7 @@ def identifier(
pattern_str = rf"{id_group_str}{{{min_digits},{max_digits}}}"
dict_ = _SimpleRegexValidator.__dict__.copy()
- dict_["regex"] = re.compile(f"^{pattern_str}$")
+ dict_["pattern"] = re.compile(f"^{pattern_str}$")
return type(
type_name,
diff --git a/src/dve/metadata_parser/function_library.py b/src/dve/metadata_parser/function_library.py
index 0e0e5af..87d0dd4 100644
--- a/src/dve/metadata_parser/function_library.py
+++ b/src/dve/metadata_parser/function_library.py
@@ -31,7 +31,7 @@ def inner(value, *args, **kwargs):
# demo function
@_nullcheck
-@pydantic.validate_arguments
+@pydantic.validate_call
def normalise(value, capitalize: bool = False): # pragma: no cover
"""Normalises a string by capitalising it"""
if capitalize:
@@ -48,7 +48,7 @@ def exclude_word(value, word: str):
@_nullcheck
-@pydantic.validate_arguments
+@pydantic.validate_call
def split(value, split_on: str, keep: int = 0):
"""Splits a string on a given delimiter and keeps only the value at the given index
defaults to 0
diff --git a/src/dve/metadata_parser/function_wrapper.py b/src/dve/metadata_parser/function_wrapper.py
index eb69546..cea12c0 100644
--- a/src/dve/metadata_parser/function_wrapper.py
+++ b/src/dve/metadata_parser/function_wrapper.py
@@ -9,13 +9,14 @@
from dve.metadata_parser import exc
PydanticCompatible = Callable[
- [Any, dict[str, Any], pydantic.fields.ModelField, pydantic.BaseConfig], Any
+ [Any, dict[str, Any], pydantic.fields.FieldInfo, pydantic.ConfigDict], # pylint: disable=E1101
+ Any,
]
"""Function Compatable with pydantic
Args:
value (Any): Value to be validated
values (dict[str, Any]): dict of previously validated fields
- field (pydantic.fields.ModelField): field object containing field name and type
+ field (pydantic.fields.FieldInfo): field object containing field name and type
config (pydantic.BaseConfig): the config that determines things like aliases
"""
@@ -24,21 +25,21 @@
def error_handler(
error_type: Union[type[Exception], type[Warning]],
error_message: str,
- field: pydantic.fields.ModelField,
+ field: pydantic.fields.FieldInfo, # pylint: disable=E1101
):
"""Determines whether to raise an error or warning based on error_type
Args:
error_type (Union[type[Exception], type[Warning]]): type of error to raise
error_message (str): message to apply
- field (pydantic.fields.ModelField): field that caused the error to be raised
+ field (pydantic.fields.FieldInfo): field that caused the error to be raised
Raises:
error_type
"""
if issubclass(error_type, exc.LocWarning):
- warnings.warn(error_type(msg=error_message, loc=field.name))
+ warnings.warn(error_type(msg=error_message, loc=field.title))
elif issubclass(error_type, Warning):
warnings.warn(error_message, error_type)
else:
@@ -90,15 +91,15 @@ def wrapper(
Returns:
Callable: wrapped function with call signature:
- (value: Any, values: dict, field: ModelField, config: BaseConfig) -> Any
+ (value: Any, values: dict, field: FieldInfo, config: BaseConfig) -> Any
"""
def inner(
value: Any,
values: dict[str, Any],
- field: pydantic.fields.ModelField, # pylint: disable=unused-argument
- config: pydantic.BaseConfig, # pylint: disable=unused-argument
+ field: pydantic.fields.FieldInfo, # pylint: disable=unused-argument,E1101
+ config: pydantic.ConfigDict, # pylint: disable=unused-argument
) -> Any:
fields = [values.get(name) for name in field_names]
result = None
@@ -120,7 +121,7 @@ def inner(
return wrapper
-validator_args = pydantic.validator.__kwdefaults__.copy()
+validator_args = pydantic.field_validator.__kwdefaults__.copy() # type: ignore
def create_validator(
@@ -153,7 +154,6 @@ def create_validator(
always: bool = False
check_fields: bool = True
whole: bool = None
- allow_reuse: bool = False
function kwargs
@@ -176,26 +176,19 @@ def create_validator(
**kwargs,
)(function)
- validator_kwargs.update(allow_reuse=True)
- validator = pydantic.validator(field, **validator_kwargs)(wrapped)
+ validator = pydantic.field_validator(field, **validator_kwargs)(wrapped)
return validator
-@pydantic.validate_arguments
+@pydantic.validate_call
def _validator_kwargs(
- pre: bool = False,
- each_item: bool = False,
- always: bool = False,
- check_fields: bool = True,
- whole: Optional[bool] = None,
- allow_reuse: bool = False,
+ mode: str = "after",
+ check_fields: bool | None = True,
+ json_schema_input_type: Any = None,
**kwargs, # pylint: disable=unused-argument
):
return {
- "pre": pre,
- "each_item": each_item,
- "always": always,
+ "mode": mode,
"check_fields": check_fields,
- "whole": whole,
- "allow_reuse": allow_reuse,
+ "json_schema_input_type": json_schema_input_type,
}
diff --git a/src/dve/metadata_parser/model_generator.py b/src/dve/metadata_parser/model_generator.py
index 2706458..892fbe1 100644
--- a/src/dve/metadata_parser/model_generator.py
+++ b/src/dve/metadata_parser/model_generator.py
@@ -37,7 +37,7 @@
"""
-@pyd.validate_arguments
+@pyd.validate_call
def constr(
*,
strip_whitespace: bool = False,
@@ -45,7 +45,6 @@ def constr(
strict: bool = False,
min_length: Optional[int] = None,
max_length: Optional[int] = None,
- curtail_length: Optional[int] = None,
regex: Optional[str] = None,
):
"""Wrapper around constr to enable argument validation"""
@@ -55,16 +54,15 @@ def constr(
strict=strict,
min_length=min_length, # type: ignore
max_length=max_length, # type: ignore
- curtail_length=curtail_length, # type: ignore
- regex=regex, # type: ignore
+ pattern=regex, # type: ignore
)
STR_TO_PY_MAPPING: Mapping[str, FieldTypeOption] = {
"constr": constr,
- "conint": pyd.validate_arguments(pyd.conint),
- "condate": pyd.validate_arguments(pyd.condate),
- "condecimal": pyd.validate_arguments(pyd.condecimal),
+ "conint": pyd.validate_call(pyd.conint),
+ "condate": pyd.validate_call(pyd.condate),
+ "condecimal": pyd.validate_call(pyd.condecimal),
"postcode": domain_types.postcode,
"nhsnumber": domain_types.NHSNumber,
"permissivenhsno": domain_types.permissive_nhs_number(),
@@ -89,7 +87,7 @@ def __init__(self, contract_contents: dict[str, Any], type_map: Optional[dict] =
@abstractmethod
def generate_models(
self, additional_validators: Optional[dict] = None
- ) -> dict[str, pyd.main.ModelMetaclass]:
+ ) -> dict[str, pyd.BaseModel]:
"""Generates models from the instance schema.
Args:
@@ -112,7 +110,7 @@ def __init__(self, contract_contents: dict[str, Any], type_map: Optional[dict] =
def generate_models(
self, additional_validators: Optional[dict] = None
- ) -> dict[str, pyd.main.ModelMetaclass]:
+ ) -> dict[str, pyd.BaseModel]:
"""Generates pydantic models from a loaded json file"""
if additional_validators:
warnings.warn("Ignoring additional validator functions")
diff --git a/src/dve/metadata_parser/models.py b/src/dve/metadata_parser/models.py
index 1d64160..fd8ed6f 100644
--- a/src/dve/metadata_parser/models.py
+++ b/src/dve/metadata_parser/models.py
@@ -5,11 +5,11 @@
import warnings
from collections import Counter
from collections.abc import Mapping, MutableMapping
-from typing import Any, Optional, Union
+from typing import Annotated, Any, Optional, Union
import pydantic as pyd
-from pydantic import BaseModel, Field, root_validator, validator
-from typing_extensions import Literal
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator
+from typing_extensions import Literal, get_origin
from dve.metadata_parser import exc, function_library
from dve.metadata_parser.function_wrapper import create_validator
@@ -27,7 +27,7 @@
"""An alias for a field."""
EntityName = str
"""The name of an entity."""
-PydanticType = Union[type, pyd.main.ModelMetaclass]
+PydanticType = Union[type, pyd.BaseModel]
"""A pydantic-appropriate type."""
ValidatorName = str
"""The name of a validator."""
@@ -63,19 +63,19 @@ class ValidationFunctionSpecification(BaseModel): # type: ignore
kwargs_: dict[str, Any] = Field(default_factory=dict, alias="kwargs")
"""Keyword arguments for the validation function."""
- @validator("name", allow_reuse=True)
- def validate_name(cls, value: str) -> str:
+ @field_validator("name")
+ def validate_name(cls, value: str, info: ValidationInfo) -> str: # pylint: disable=W0613
"""Ensure that the name exists in the function library."""
if not hasattr(function_library, value):
raise ValueError(f"Function {value!r} not available in function library")
return value
- @validator("error_message", allow_reuse=True)
- def validate_error_message(cls, value: str, values: dict[str, Any]) -> str:
+ @field_validator("error_message")
+ def validate_error_message(cls, value: str, info: ValidationInfo) -> str:
"""set a default error message if one is not available."""
if value:
return value
- name: str = values["name"]
+ name: str = info.data["name"]
return f"{name} failed"
def get_field_validator(self, field_name: str, **extra_kwargs: Any) -> classmethod:
@@ -128,12 +128,15 @@ class FieldSpecification(BaseModel):
functions: list[ValidationFunctionSpecification] = Field(default_factory=list)
"""Validation functions to be applied to the type."""
- @root_validator(allow_reuse=True)
- def ensure_one_type_spec_method(cls, values: dict[str, Any]) -> dict[str, Any]:
+ model_config = {"validate_assignment": True}
+
+ @model_validator(mode="after") # type: ignore
+ @classmethod
+ def ensure_one_type_spec_method(cls, field_spec) -> dict[str, Any]:
"""Ensure that exactly one of 'type', 'model' and 'callable' was specified."""
- has_type = bool(values.get("type_"))
- has_model = bool(values.get("model"))
- has_callable = bool(values.get("callable"))
+ has_type = bool(field_spec.type_)
+ has_model = bool(field_spec.model)
+ has_callable = bool(field_spec.callable)
n_specified = sum((has_type, has_model, has_callable))
failure_messages = [
@@ -156,21 +159,21 @@ def ensure_one_type_spec_method(cls, values: dict[str, Any]) -> dict[str, Any]:
failure_messages.append(f"Got {supplied}")
raise ValueError(" ".join(failure_messages))
- if not has_callable and values.get("constraints"):
+ if not has_callable and field_spec.constraints:
warnings.warn(
"'constraints' only used when field specification uses 'callable'",
category=UnusedConstraints,
)
- return values
+ return field_spec
- @validator("default", allow_reuse=True)
- def validate_default(cls, value: Any, values: dict[str, Any]) -> Any:
+ @field_validator("default")
+ def validate_default(cls, value: Any, info: ValidationInfo) -> Any:
"""Validate that 'default' is aligned with 'is_array'."""
if value is None:
return value
- is_array = bool(values.get("is_array"))
+ is_array = bool(info.data.get("is_array"))
if is_array:
if not isinstance(value, list):
warnings.warn(
@@ -212,7 +215,7 @@ def get_type_and_validators(
self,
field_name: str,
*type_mappings: Mapping[TypeName, FieldTypeOption],
- schemas: Optional[dict[EntityName, pyd.main.ModelMetaclass]] = None,
+ schemas: Optional[dict[EntityName, pyd.BaseModel]] = None,
is_mandatory: bool = False,
) -> tuple[PydanticType, Default, Validators]:
"""Get the type, default value, and validators for the specification."""
@@ -223,12 +226,14 @@ def get_type_and_validators(
possible_python_type = chain_get(self.type_, *type_mappings, pyd, dt, __builtins__)
if isinstance(possible_python_type, type):
python_type = possible_python_type
+ elif get_origin(possible_python_type) is Annotated:
+ python_type = possible_python_type # type: ignore
elif hasattr(possible_python_type, "get_type_and_validators"):
possible_python_type: "FieldSpecification" # type: ignore
nested_vals = possible_python_type.get_type_and_validators( # type: ignore
field_name, *type_mappings, schemas=schemas, is_mandatory=False
)
- python_type, nested_default, nested_validators = nested_vals
+ python_type, nested_default, nested_validators = nested_vals # type: ignore
if nested_validators and self.is_array:
# Need to work out how to hook into the validators and update
@@ -248,7 +253,7 @@ def get_type_and_validators(
if not schemas:
raise ValueError("Type should be model, but `schemas` not passed")
try:
- python_type = schemas[self.model]
+ python_type = schemas[self.model] # type: ignore
except KeyError as err:
raise ValueError(
f"Type should be model {self.model!r} but this is not in `schemas`"
@@ -262,15 +267,20 @@ def get_type_and_validators(
raise ValueError("No field type set")
default = default or (... if is_mandatory else None)
+
if self.is_array:
python_type = list[python_type] # type: ignore
+
+ if not is_mandatory:
+ python_type = Optional[python_type] # type: ignore
+
return python_type, default, validators
class EntitySpecification(BaseModel):
"""Configuration options for an entity."""
- fields: dict[FieldName, FieldSpecification]
+ fields: Annotated[dict[FieldName, FieldSpecification], Field(validate_default=True)]
"""
A mapping of field names to their Python types. These will either be
strings representing Python types (if there are no argumements to the type),
@@ -282,9 +292,11 @@ class EntitySpecification(BaseModel):
mandatory_fields: list[FieldName] = Field(default_factory=list)
"""An array of field names which should be considered mandatory."""
- @validator("fields", pre=True, allow_reuse=True)
+ @field_validator("fields", mode="before")
def validate_fields(
- cls, value: dict[FieldName, Union[TypeName, FieldSpecification]]
+ cls,
+ value: dict[FieldName, Union[TypeName, FieldSpecification]],
+ info: ValidationInfo, # pylint: disable=W0613
) -> dict[FieldName, FieldSpecification]:
"""Convert bare string fields to field specifications."""
for key in value:
@@ -294,9 +306,9 @@ def validate_fields(
return value # type: ignore
- @validator("aliases", allow_reuse=True)
+ @field_validator("aliases")
def validate_aliases(
- cls, value: dict[FieldName, FieldAlias], values: dict[str, Any]
+ cls, value: dict[FieldName, FieldAlias], info: ValidationInfo
) -> dict[FieldName, FieldAlias]:
"""Ensure that 'aliases' is aligned with 'fields'."""
# Check that aliases are not given more than once
@@ -316,7 +328,7 @@ def validate_aliases(
+ f"more than once: {multiple_occurrences}"
)
# And warn when unnecessary aliases were given.
- field_names: set[FieldName] = set(values["fields"].keys())
+ field_names: set[FieldName] = set(info.data["fields"].keys())
missing_fields = set(value.keys()) - field_names
if missing_fields:
warnings.warn(
@@ -327,15 +339,15 @@ def validate_aliases(
return value
- @validator("mandatory_fields", allow_reuse=True)
+ @field_validator("mandatory_fields")
def validate_mandatory_fields(
- cls, value: list[FieldName], values: dict[str, Any]
+ cls, value: list[FieldName], info: ValidationInfo
) -> list[FieldName]:
"""Ensure that 'mandatory_fields' is aligned with 'fields'."""
if not value:
return value
- field_names: set[FieldName] = set(values["fields"].keys())
+ field_names: set[FieldName] = set(info.data["fields"].keys())
missing_fields = set(value) - field_names
if missing_fields:
raise ValueError(
@@ -349,8 +361,8 @@ def as_model(
self,
model_name: str,
*type_mappings: Mapping[TypeName, FieldTypeOption],
- schemas: Optional[dict[EntityName, pyd.main.ModelMetaclass]] = None,
- ) -> pyd.main.ModelMetaclass:
+ schemas: Optional[dict[EntityName, pyd.BaseModel]] = None,
+ ) -> pyd.BaseModel:
"""Get the pydantic model from an entity definition."""
validators = {}
pyd_fields = {}
@@ -365,18 +377,12 @@ def as_model(
pyd_fields[field_name] = (python_type, default)
validators.update(field_validators)
- class Config(pyd.BaseConfig):
- """Model configuration."""
-
- fields = self.aliases # type: ignore
- anystr_strip_whitespace = True
- allow_population_by_field_name = True
- extra = pyd.Extra.ignore
-
return pyd.create_model( # type: ignore
model_name,
**pyd_fields,
- __config__=Config, # type: ignore
+ __config__=ConfigDict(
+ str_strip_whitespace=True, populate_by_name=True, extra="ignore"
+ ), # type: ignore
__validators__=validators,
)
@@ -395,9 +401,9 @@ class DatasetSpecification(BaseModel):
def load_models(
self,
*type_mappings: Mapping[TypeName, FieldTypeOption],
- ) -> dict[EntityName, pyd.main.ModelMetaclass]:
+ ) -> dict[EntityName, pyd.BaseModel]:
"""Load the models from the dataset definition."""
- loaded_schemas: dict[EntityName, pyd.main.ModelMetaclass] = {}
+ loaded_schemas: dict[EntityName, pyd.BaseModel] = {}
for model_name, specification in self.schemas.items(): # pylint: disable=E1101
loaded_schemas[model_name] = specification.as_model(
model_name, self.types, *type_mappings, schemas=loaded_schemas
diff --git a/src/dve/parser/file_handling/implementations/file.py b/src/dve/parser/file_handling/implementations/file.py
index 76d8b58..9fb29db 100644
--- a/src/dve/parser/file_handling/implementations/file.py
+++ b/src/dve/parser/file_handling/implementations/file.py
@@ -2,7 +2,7 @@
import platform
import shutil
-from collections.abc import Callable, Iterator
+from collections.abc import Iterator
from pathlib import Path
from typing import IO, Any, NoReturn, Optional
from urllib.parse import unquote
@@ -12,7 +12,7 @@
from dve.parser.exceptions import FileAccessError
from dve.parser.file_handling.helpers import parse_uri
from dve.parser.file_handling.implementations.base import BaseFilesystemImplementation
-from dve.parser.type_hints import URI, NodeType, PathStr, Scheme
+from dve.parser.type_hints import URI, NodeType, Scheme
FILE_URI_SCHEMES: set[Scheme] = {"file"}
"""A set of all allowed file URI schemes."""
@@ -179,9 +179,9 @@ def _transfer_resource(
parent.mkdir(exist_ok=True)
if action == "copy":
- func: Callable[[PathStr, PathStr], None] = shutil.copy
+ func = shutil.copy # type: ignore
elif action == "move":
- func = shutil.move
+ func = shutil.move # type: ignore
else: # pragma: no cover
raise ValueError(f"Unsupported action {action!r}, expected one of: 'copy', 'move'")
try:
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index 67fdf88..8c32021 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -13,7 +13,7 @@
from uuid import uuid4
import polars as pl
-from pydantic import validate_arguments
+from pydantic import validate_call
import dve.reporting.excel_report as er
from dve.common.error_utils import (
@@ -139,7 +139,7 @@ def get_submission_status(
return SubmissionStatus()
return submission_status
- @validate_arguments
+ @validate_call
def _move_submission_to_working_location(
self,
submission_id: str,
@@ -824,7 +824,7 @@ def error_report(
summary_dict = {
key.replace("_", " ").title(): value
- for key, value in submission_info.dict().items()
+ for key, value in submission_info.model_dump().items()
if value is not None and not key.endswith("_updated")
}
summary_items = er.SummaryItems(
diff --git a/src/dve/pipeline/utils.py b/src/dve/pipeline/utils.py
index eebaa90..4fe114e 100644
--- a/src/dve/pipeline/utils.py
+++ b/src/dve/pipeline/utils.py
@@ -5,7 +5,7 @@
from threading import Lock
from typing import Any, Optional
-from pydantic.main import ModelMetaclass
+from pydantic import BaseModel
from pyspark.sql import SparkSession
import dve.core_engine.backends.implementations.duckdb # pylint: disable=unused-import
@@ -18,7 +18,7 @@
from dve.metadata_parser.model_generator import JSONtoPyd
Dataset = dict[SchemaName, _ModelConfig]
-_configs: dict[str, tuple[dict[str, ModelMetaclass], V1EngineConfig, Dataset]] = {}
+_configs: dict[str, tuple[dict[str, BaseModel], V1EngineConfig, Dataset]] = {}
locks = Lock()
logger = get_logger(__name__)
@@ -27,7 +27,7 @@
def load_config(
dataset_id: str,
file_uri: URI,
-) -> tuple[dict[SchemaName, ModelMetaclass], V1EngineConfig, dict[SchemaName, _ModelConfig]]:
+) -> tuple[dict[SchemaName, BaseModel], V1EngineConfig, dict[SchemaName, _ModelConfig]]:
"""Loads the configuration for a given dataset"""
if dataset_id in _configs:
return _configs[dataset_id]
diff --git a/tests/features/steps/steps_post_pipeline.py b/tests/features/steps/steps_post_pipeline.py
index be679ba..906445e 100644
--- a/tests/features/steps/steps_post_pipeline.py
+++ b/tests/features/steps/steps_post_pipeline.py
@@ -108,7 +108,7 @@ def check_stats_record(context):
record: Dict[str, str] = row.as_dict()
expected[record["parameter"]] = int(record["value"])
stats = (
- get_pipeline(context)._audit_tables.get_submission_statistics(sub_info.submission_id).dict()
+ get_pipeline(context)._audit_tables.get_submission_statistics(sub_info.submission_id).model_dump()
)
assert all([val == stats.get(fld) for fld, val in expected.items()])
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 457cf42..fa79eec 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -12,9 +12,6 @@
from moto import mock_s3 # type: ignore
from pyspark.sql import SparkSession
-from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
- PYTHON_TYPE_TO_DUCKDB_TYPE,
-)
from dve.parser.file_handling.implementations import DBFSFilesystemImplementation
from dve.parser.file_handling.service import (
add_implementation,
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py
index 5daa471..3898ad5 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py
@@ -344,11 +344,11 @@ def test_get_error_report_submissions(ddb_audit_manager_threaded: DDBAuditingMan
expected = [
SubmissionInfo(
submission_id=sub_1.submission_id,
- **{fld: val for fld, val in sub_1.dict().items() if fld != "submission_id"},
+ **{fld: val for fld, val in sub_1.model_dump().items() if fld != "submission_id"},
),
SubmissionInfo(
submission_id=sub_3.submission_id,
- **{fld: val for fld, val in sub_3.dict().items() if fld != "submission_id"},
+ **{fld: val for fld, val in sub_3.model_dump().items() if fld != "submission_id"},
),
]
assert len(processed) == 2
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py
index 74edce9..2019a66 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py
@@ -96,8 +96,8 @@ def test_duckdb_data_contract_csv(temp_csv_file):
entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(uri.as_posix()), entities, entity_locations, dc_meta)
rel: DuckDBPyRelation = entities.get("test_ds")
expected_schema = {
- fld.name: str(get_duckdb_type_from_annotation(fld.annotation))
- for fld in mdl.__fields__.values()
+ name: str(get_duckdb_type_from_annotation(fld.annotation))
+ for name, fld in mdl.model_fields.items()
}
expected_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int)
assert dict(zip(rel.columns, rel.dtypes)) == expected_schema
@@ -196,13 +196,13 @@ def test_duckdb_data_contract_xml(temp_xml_file):
entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(uri.as_posix()), entities, entity_locations, dc_meta)
header_rel: DuckDBPyRelation = entities.get("test_header")
header_expected_schema: Dict[str, DuckDBPyType] = {
- fld.name: get_duckdb_type_from_annotation(fld.type_)
- for fld in header_model.__fields__.values()
+ name: get_duckdb_type_from_annotation(fld.annotation)
+ for name, fld in header_model.model_fields.items()
}
header_expected_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int)
class_data_expected_schema: Dict[str, DuckDBPyType] = {
- fld.name: get_duckdb_type_from_annotation(fld.type_)
- for fld in class_model.__fields__.values()
+ name: get_duckdb_type_from_annotation(fld.annotation)
+ for name, fld in class_model.model_fields.items()
}
class_data_expected_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int)
class_data_rel: DuckDBPyRelation = entities.get("test_class_info")
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
index 19e96e2..87e668f 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
@@ -168,9 +168,9 @@ def test_get_duckdb_cast_statement_from_annotation(field_name, field_type, cast_
def test_use_cast_statements(casting_test_table):
_, conn = casting_test_table
test_rel = conn.sql("SELECT * from test_casting")
- casting_statements = [ f"{get_duckdb_cast_statement_from_annotation(fld.name, fld.annotation)} as {fld.name}" for fld in CastingRecord.__fields__.values()]
+ casting_statements = [ f"{get_duckdb_cast_statement_from_annotation(name, fld.annotation)} as {name}" for name, fld in CastingRecord.model_fields.items()]
test_rel = test_rel.project(",".join(casting_statements))
- assert dict(zip(test_rel.columns, test_rel.dtypes)) == {fld.name: get_duckdb_type_from_annotation(fld.annotation) for fld in CastingRecord.__fields__.values()}
+ assert dict(zip(test_rel.columns, test_rel.dtypes)) == {name: get_duckdb_type_from_annotation(fld.annotation) for name, fld in CastingRecord.model_fields.items()}
dodgy_date_rec = test_rel.pl()[1].to_dicts()[0]
assert (not dodgy_date_rec.get("date_test") and
not dodgy_date_rec.get("basic_model",{}).get("date_field")
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
index 5d3d3a8..35007a9 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
@@ -69,7 +69,7 @@ def test_column_addition(planets_rel: DuckDBPyRelation):
rule = ColumnAddition(
entity_name="planets",
column_name="literal_one",
- expression=1,
+ expression="1",
new_entity_name="added",
)
_, success = DUCKDB_STEP_BACKEND.evaluate(entities, config=rule)
@@ -87,7 +87,7 @@ def test_column_addition_missing_entity():
rule = ColumnAddition(
entity_name="planets",
column_name="literal_one",
- expression=1,
+ expression="1",
new_entity_name="added",
)
messages, success = DUCKDB_STEP_BACKEND.evaluate(entities, config=rule)
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py
index 4b328df..7da7f48 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py
@@ -359,11 +359,11 @@ def test_get_error_report_submissions(spark_audit_manager: SparkAuditingManager)
expected = [
SubmissionInfo(
submission_id=sub_1.submission_id,
- **{fld: val for fld, val in sub_1.dict().items() if fld != "submission_id"},
+ **{fld: val for fld, val in sub_1.model_dump().items() if fld != "submission_id"},
),
SubmissionInfo(
submission_id=sub_3.submission_id,
- **{fld: val for fld, val in sub_3.dict().items() if fld != "submission_id"},
+ **{fld: val for fld, val in sub_3.model_dump().items() if fld != "submission_id"},
),
]
assert len(processed) == 2
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py
index e0cbf90..673e611 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py
@@ -59,7 +59,7 @@ def test_column_addition(planets_df: DataFrame):
rule = ColumnAddition(
entity_name="planets",
column_name="literal_one",
- expression=1,
+ expression="1",
new_entity_name="added",
)
_, success = SPARK_STEP_BACKEND.evaluate(entities, config=rule)
@@ -77,7 +77,7 @@ def test_column_addition_missing_entity():
rule = ColumnAddition(
entity_name="planets",
column_name="literal_one",
- expression=1,
+ expression="1",
new_entity_name="added",
)
messages, success = SPARK_STEP_BACKEND.evaluate(entities, config=rule)
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
index 7502673..2a336cb 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
@@ -256,9 +256,9 @@ def test_get_spark_cast_statement_from_annotation(field_name, field_type, expres
def test_use_cast_statements(spark, casting_dataframe):
- casting_statements = [ get_spark_cast_statement_from_annotation(fld.name, fld.annotation).alias(fld.name) for fld in CastingRecord.__fields__.values()]
+ casting_statements = [ get_spark_cast_statement_from_annotation(name, fld.annotation).alias(name) for name, fld in CastingRecord.model_fields.items()]
cast_df = casting_dataframe.select(*casting_statements)
- assert {fld.name: fld.dataType for fld in cast_df.schema} == {fld.name: get_type_from_annotation(fld.annotation) for fld in CastingRecord.__fields__.values()}
+ assert {fld.name: fld.dataType for fld in cast_df.schema} == {name: get_type_from_annotation(fld.annotation) for name, fld in CastingRecord.model_fields.items()}
dodgy_date_rec = [rw.asDict(True) for rw in cast_df.collect()][1]
assert (not dodgy_date_rec.get("date_test") and
not dodgy_date_rec.get("basic_model",{}).get("date_field")
diff --git a/tests/test_core_engine/test_backends/test_readers/test_csv.py b/tests/test_core_engine/test_backends/test_readers/test_csv.py
index 0737ad2..c7b2d6a 100644
--- a/tests/test_core_engine/test_backends/test_readers/test_csv.py
+++ b/tests/test_core_engine/test_backends/test_readers/test_csv.py
@@ -140,7 +140,7 @@ def test_csv_file_get_subset(
parsed = {row["planet"]: row for row in results}
# Keep only keys in the subset from the source
- subset_keys = set(PlanetsSubset.__fields__.keys())
+ subset_keys = set(PlanetsSubset.model_fields.keys())
for data in planet_data.values():
to_pop = set(data.keys()) - subset_keys - {RECORD_INDEX_COLUMN_NAME}
for key in to_pop:
@@ -162,7 +162,7 @@ def test_csv_file_get_subset_add_missing(
parsed = {row["planet"]: row for row in results}
# Keep only keys in the subset from the source
- subset_keys = set(PlanetsSubset.__fields__.keys())
+ subset_keys = set(PlanetsSubset.model_fields.keys())
for data in planet_data.values():
to_pop = set(data.keys()) - subset_keys - {RECORD_INDEX_COLUMN_NAME}
for key in to_pop:
diff --git a/tests/test_core_engine/test_backends/test_readers/test_ddb_csv.py b/tests/test_core_engine/test_backends/test_readers/test_ddb_csv.py
index 78974eb..96bc5f7 100644
--- a/tests/test_core_engine/test_backends/test_readers/test_ddb_csv.py
+++ b/tests/test_core_engine/test_backends/test_readers/test_ddb_csv.py
@@ -88,8 +88,8 @@ def test_ddb_csv_reader_cast(temp_csv_file):
reader = DuckDBCSVReader(header=True, delim=",", connection=duckdb.connect())
rel: DuckDBPyRelation = reader.read_to_entity_type(DuckDBPyRelation, str(uri), "test", mdl)
expected_dtypes = {**{
- fld.name: str(get_duckdb_type_from_annotation(fld.annotation))
- for fld in mdl.__fields__.values()
+ name: str(get_duckdb_type_from_annotation(fld.annotation))
+ for name, fld in mdl.model_fields.items()
}, RECORD_INDEX_COLUMN_NAME: get_duckdb_type_from_annotation(int)}
expected_data = [(*rw, idx) for idx, rw in enumerate(data, start=1)]
assert rel.columns == header.split(",") + [RECORD_INDEX_COLUMN_NAME]
diff --git a/tests/test_core_engine/test_backends/test_readers/test_ddb_json.py b/tests/test_core_engine/test_backends/test_readers/test_ddb_json.py
index 87a90e8..ae8e4ad 100644
--- a/tests/test_core_engine/test_backends/test_readers/test_ddb_json.py
+++ b/tests/test_core_engine/test_backends/test_readers/test_ddb_json.py
@@ -56,7 +56,7 @@ class SimpleModel(BaseModel):
def test_ddb_json_reader_all_str(temp_json_file):
uri, data, mdl = temp_json_file
- expected_fields = [fld for fld in mdl.__fields__]
+ expected_fields = [fld for fld in mdl.model_fields]
reader = DuckDBJSONReader()
rel: DuckDBPyRelation = reader.read_to_entity_type(
DuckDBPyRelation, uri.as_posix(), "test", stringify_model(mdl)
@@ -68,14 +68,14 @@ def test_ddb_json_reader_all_str(temp_json_file):
def test_ddb_json_reader_cast(temp_json_file):
uri, data, mdl = temp_json_file
- expected_fields = [fld for fld in mdl.__fields__]
+ expected_fields = [fld for fld in mdl.model_fields]
reader = DuckDBJSONReader()
rel: DuckDBPyRelation = reader.read_to_entity_type(DuckDBPyRelation, uri.as_posix(), "test", mdl)
assert rel.columns == expected_fields + [RECORD_INDEX_COLUMN_NAME]
assert dict(zip(rel.columns, rel.dtypes)) == {**{
- fld.name: str(get_duckdb_type_from_annotation(fld.annotation))
- for fld in mdl.__fields__.values()
+ name: str(get_duckdb_type_from_annotation(fld.annotation))
+ for name, fld in mdl.model_fields.items()
}, RECORD_INDEX_COLUMN_NAME: "BIGINT"}
assert rel.fetchall() == [(*rw.values(), idx) for idx, rw in enumerate(data, start = 1)]
diff --git a/tests/test_core_engine/test_backends/test_readers/test_spark_json.py b/tests/test_core_engine/test_backends/test_readers/test_spark_json.py
index 24674ca..46d4729 100644
--- a/tests/test_core_engine/test_backends/test_readers/test_spark_json.py
+++ b/tests/test_core_engine/test_backends/test_readers/test_spark_json.py
@@ -55,7 +55,7 @@ class SimpleModel(BaseModel):
def test_spark_json_reader_all_str(temp_json_file):
uri, data, mdl = temp_json_file
- expected_fields = [fld for fld in mdl.__fields__] + [RECORD_INDEX_COLUMN_NAME]
+ expected_fields = [fld for fld in mdl.model_fields] + [RECORD_INDEX_COLUMN_NAME]
reader = SparkJSONReader()
df: DataFrame = reader.read_to_entity_type(
DataFrame, uri.as_posix(), "test", stringify_model(mdl)
@@ -66,13 +66,13 @@ def test_spark_json_reader_all_str(temp_json_file):
def test_spark_json_reader_cast(temp_json_file):
uri, data, mdl = temp_json_file
- expected_fields = [fld for fld in mdl.__fields__] + [RECORD_INDEX_COLUMN_NAME]
+ expected_fields = [fld for fld in mdl.model_fields] + [RECORD_INDEX_COLUMN_NAME]
reader = SparkJSONReader()
df: DataFrame = reader.read_to_entity_type(DataFrame, uri.as_posix(), "test", mdl)
assert df.columns == expected_fields
- assert df.schema == StructType([StructField(fld.name, get_type_from_annotation(fld.annotation))
- for fld in mdl.__fields__.values()] + [StructField(RECORD_INDEX_COLUMN_NAME, get_type_from_annotation(int))])
+ assert df.schema == StructType([StructField(name, get_type_from_annotation(fld.annotation))
+ for name, fld in mdl.model_fields.items()] + [StructField(RECORD_INDEX_COLUMN_NAME, get_type_from_annotation(int))])
assert [rw.asDict() for rw in df.collect()] == [{**rw, RECORD_INDEX_COLUMN_NAME: idx} for idx, rw in enumerate(data, start=1)]
diff --git a/tests/test_core_engine/test_backends/test_utilities.py b/tests/test_core_engine/test_backends/test_utilities.py
new file mode 100644
index 0000000..839d9b7
--- /dev/null
+++ b/tests/test_core_engine/test_backends/test_utilities.py
@@ -0,0 +1,55 @@
+from datetime import date
+from typing import Any
+
+import pytest
+from pydantic import BaseModel
+
+from dve.core_engine.backends.utilities import is_field_complex
+
+
+class AnotherTestModel(BaseModel):
+ test_field: str
+
+
+class TestModel(BaseModel):
+ simple_str: str
+ simple_int: int
+ simple_bool: bool
+ simple_list: list[str]
+ simple_dict: dict[str, Any]
+ simple_set: set[str]
+ simple_tuple: tuple[int, str]
+ another_model: AnotherTestModel
+ date_example: date
+
+
+TEST_MODEL = TestModel(
+ simple_str="abc",
+ simple_int=123,
+ simple_bool=True,
+ simple_list=["apple", "banana", "orange",],
+ simple_dict={"key1": "abc"},
+ simple_set={"a", "b", "c"},
+ simple_tuple=(1, "wow"),
+ another_model=AnotherTestModel(test_field="lemon"),
+ date_example=date(2026,1,1)
+)
+
+
+@pytest.mark.parametrize(
+ "field_name, expected",
+ [
+ ("simple_str", False),
+ ("simple_int", False),
+ ("simple_bool", False),
+ ("simple_list", True),
+ ("simple_dict", True),
+ ("simple_set", True),
+ ("simple_tuple", True),
+ ("another_model", True),
+ ]
+)
+def test_is_field_complex(field_name: str, expected: bool):
+ _field = TEST_MODEL.model_fields[field_name]
+ _res = is_field_complex(_field)
+ assert _res == expected, f"Expected {expected}. Got {_res}."
diff --git a/tests/test_core_engine/test_message.py b/tests/test_core_engine/test_message.py
index ccb6736..9a4f8ac 100644
--- a/tests/test_core_engine/test_message.py
+++ b/tests/test_core_engine/test_message.py
@@ -4,8 +4,9 @@
import json
from string import ascii_letters
from typing import Dict, List, Optional
+from typing_extensions import Annotated
-from pydantic import BaseModel, ValidationError
+from pydantic import BaseModel, Field, ValidationError
import pytest
from dve.core_engine.message import DEFAULT_ERROR_DETAIL, DataContractErrorDetail, FeedbackMessage
@@ -140,7 +141,7 @@ def test_from_pydantic_error():
class TestModel(BaseModel):
idx: int
- other_field: str
+ other_field: Annotated[str, Field(coerce_numbers_to_str=True)]
_bad_value_data = {"idx": "ABC", "other_field": 123}
_blank_value_data = {"other_field": "hi"}
@@ -178,9 +179,9 @@ def test_from_pydantic_error_custom_error_details():
class TestModel(BaseModel):
idx: int
str_field: str
- date_field: Optional[date]
- unimportant_field: Optional[int]
-
+ date_field: Annotated[date, Field(default=None)]
+ unimportant_field: Annotated[int, Field(default=None)]
+
custom_error_details: str = """
{"idx": {"Blank": {"error_code": "IDBLANKERRCODE",
"error_message": "idx is a mandatory field"},
@@ -239,7 +240,7 @@ def test_from_pydantic_error_custom_codes_nested():
class LowestModel(BaseModel):
nested_field_3: str
- test_date: Optional[date]
+ test_date: Annotated[date, Field(default=None)]
class SubTestModel(BaseModel):
nested_field_1: int
nested_field_2: List[LowestModel]
diff --git a/tests/test_core_engine/test_models.py b/tests/test_core_engine/test_models.py
index e187de1..d1a55c4 100644
--- a/tests/test_core_engine/test_models.py
+++ b/tests/test_core_engine/test_models.py
@@ -1,10 +1,11 @@
"""Unit tests for core engine models."""
+from datetime import date, datetime
from typing import Any, Dict, Tuple
from uuid import uuid4
import pytest
-from dve.core_engine.models import SubmissionInfo
+from dve.core_engine.models import AuditRecord, SubmissionInfo
CONSTANT_SUBMISSION_ID = uuid4().hex
@@ -116,7 +117,7 @@ def test_submission_info( # pylint: disable=missing-function-docstring
actual = SubmissionInfo(**testcase["submitted"])
expected = testcase["expected"]
- assert {k: v for k, v in actual.dict().items() if k not in ignore} == expected
+ assert {k: v for k, v in actual.model_dump().items() if k not in ignore} == expected
assert actual.file_name_with_ext == f"{expected['file_name']}.{expected['file_extension']}"
@@ -128,3 +129,23 @@ def test_submission_info_eq(): # pylint: disable=missing-function-docstring
"file_extension": "csv",
}
assert SubmissionInfo(**data) == SubmissionInfo(**data) # type: ignore
+
+
+class TestAuditRecord:
+ submission_id = uuid4().hex
+
+ def test_audit_record_just_time_updated(self):
+ data = {
+ "submission_id": self.submission_id,
+ "time_updated": datetime(2025,12,1,12,30,10)
+ }
+
+ assert AuditRecord(**data).date_updated == date(2025,12,1)
+
+ def test_audit_record_date_updated(self):
+ data = {
+ "submission_id": self.submission_id,
+ "date_updated": date(2025,12,1)
+ }
+
+ assert AuditRecord(**data).date_updated == date.today()
diff --git a/tests/test_model_generation/test_domain_types.py b/tests/test_model_generation/test_domain_types.py
index 56cf3f9..8917bac 100644
--- a/tests/test_model_generation/test_domain_types.py
+++ b/tests/test_model_generation/test_domain_types.py
@@ -15,10 +15,10 @@
class ATestModel(BaseModel):
- nhsnumber: Optional[hct.NHSNumber]
- postcode: Optional[hct.Postcode]
- org_id: Optional[hct.OrgID]
- nhsnumber2: Optional[hct.permissive_nhs_number()]
+ nhsnumber: Optional[hct.NHSNumber] = None
+ postcode: Optional[hct.Postcode] = None
+ org_id: Optional[hct.OrgID] = None
+ nhsnumber2: Optional[hct.permissive_nhs_number()] = None
class DatetimeModel(BaseModel):
@@ -26,8 +26,8 @@ class DatetimeModel(BaseModel):
class ReportingPeriodModel(BaseModel):
- reporting_period_start: Optional[hct.reportingperiod(reporting_period_type="start")]
- reporting_period_end: Optional[hct.reportingperiod(reporting_period_type="end")]
+ reporting_period_start: Optional[hct.reportingperiod(reporting_period_type="start")] = None
+ reporting_period_end: Optional[hct.reportingperiod(reporting_period_type="end")] = None
@pytest.mark.parametrize(
@@ -256,9 +256,9 @@ def test_formatteddatetime_in_model_raises(datetime_to_validate: Union[str, dt.d
class DateModel(BaseModel):
"""Model for testing FormattedDate."""
- formatted_date: Optional[hct.conformatteddate("%Y-%m-%d")]
- formatted_date_constrained: Optional[hct.conformatteddate("%Y-%m-%d", ge="1970-01-01")]
- formatted_date_constrained2: Optional[hct.conformatteddate("%Y-%m-%d", lt="1970-01-01")]
+ formatted_date: Optional[hct.conformatteddate("%Y-%m-%d")] = None
+ formatted_date_constrained: Optional[hct.conformatteddate("%Y-%m-%d", ge="1970-01-01")] = None
+ formatted_date_constrained2: Optional[hct.conformatteddate("%Y-%m-%d", lt="1970-01-01")] = None
@pytest.mark.parametrize(
diff --git a/tests/test_pipeline/test_spark_pipeline.py b/tests/test_pipeline/test_spark_pipeline.py
index b3048a1..974f4ac 100644
--- a/tests/test_pipeline/test_spark_pipeline.py
+++ b/tests/test_pipeline/test_spark_pipeline.py
@@ -166,7 +166,7 @@ def test_apply_data_contract_failed( # pylint: disable=redefined-outer-name
"Key": "",
"FailureType": "record",
"Status": "error",
- "ErrorType": "value_error.any_str.max_length",
+ "ErrorType": "string_too_long",
"ErrorLocation": "planet",
"ErrorMessage": "is invalid",
"ErrorCode": "BadValue",
@@ -180,7 +180,7 @@ def test_apply_data_contract_failed( # pylint: disable=redefined-outer-name
"Key": "",
"FailureType": "record",
"Status": "error",
- "ErrorType": "value_error.number.not_ge",
+ "ErrorType": "greater_than_equal",
"ErrorLocation": "numberOfMoons",
"ErrorMessage": "is invalid",
"ErrorCode": "BadValue",
@@ -194,7 +194,7 @@ def test_apply_data_contract_failed( # pylint: disable=redefined-outer-name
"Key": "",
"FailureType": "record",
"Status": "error",
- "ErrorType": "type_error.bool",
+ "ErrorType": "bool_parsing",
"ErrorLocation": "hasGlobalMagneticField",
"ErrorMessage": "is invalid",
"ErrorCode": "BadValue",
@@ -421,7 +421,7 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out
"number_warnings": 0,
}
- sub_stats = stats.dict()
+ sub_stats = stats.model_dump() if stats else {}
assert all([expected.get(key) == sub_stats.get(key) for key in expected])
From d27fb21442b3781bb430a8e74990cfbdfe8a0227 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 5 Jun 2026 13:29:40 +0100
Subject: [PATCH 02/17] build: Upgrade poetry version to v2.4.1 and lock boto3
and botocore dependencies (#110)
added `min-release-age` argument to poetry.toml to ensure dependencies are at least 10 days old before upgrading. This was made available in poetry v2.4
---
.mise.toml | 2 +-
CONTRIBUTE.md | 2 +-
Makefile | 5 -----
poetry.lock | 8 ++++----
poetry.toml | 3 +++
pyproject.toml | 4 ++--
6 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/.mise.toml b/.mise.toml
index d41748e..be3149d 100644
--- a/.mise.toml
+++ b/.mise.toml
@@ -1,4 +1,4 @@
[tools]
python="3.12"
-poetry="2.3.3"
+poetry="2.4.1"
java="liberica-1.8.0"
diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md
index b5d1eed..d300ec4 100644
--- a/CONTRIBUTE.md
+++ b/CONTRIBUTE.md
@@ -58,7 +58,7 @@ poetry install --with lint
## Installation for Development
-We are utilising Poetry for build dependency management and packaging. If you're on a system that has `Make` available, you can simply run `make install` to setup a local virtual environment with all the dependencies installed (this won't install Poetry for you).
+We are utilising Poetry for build dependency management and packaging. To install simply run `poetry install`. If you need to update any of the dependencies then you will need to run `poetry lock` before the install command. **Please always review the contents of the lock before installing. We have a `min-release-age` of `10` which means that packages will not be updated until they have been released for at least 10 days.
## Testing
diff --git a/Makefile b/Makefile
index 7f31aa7..50bf37c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,5 @@
activate = poetry run
-# dev
-install:
- poetry lock
- poetry install --with dev
-
# dist
wheel:
poetry build -f wheel
diff --git a/poetry.lock b/poetry.lock
index ec61a6d..b297876 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand.
[[package]]
name = "annotated-types"
@@ -2210,8 +2210,8 @@ files = [
[package.dependencies]
numpy = [
{version = ">=1.22.4", markers = "python_version < \"3.11\""},
- {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
{version = ">=1.23.2", markers = "python_version == \"3.11\""},
+ {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
]
python-dateutil = ">=2.8.2"
pytz = ">=2020.1"
@@ -2685,8 +2685,8 @@ astroid = ">=3.3.8,<=3.4.0.dev0"
colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""}
dill = [
{version = ">=0.2", markers = "python_version < \"3.11\""},
- {version = ">=0.3.7", markers = "python_version >= \"3.12\""},
{version = ">=0.3.6", markers = "python_version == \"3.11\""},
+ {version = ">=0.3.7", markers = "python_version >= \"3.12\""},
]
isort = ">=4.2.5,<5.13 || >5.13,<7"
mccabe = ">=0.6,<0.8"
@@ -3541,4 +3541,4 @@ type = ["pytest-mypy"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.13"
-content-hash = "793e65e9dcefec411b9423ed3587c05121529b422f9299ac99cd157c7dec7c01"
+content-hash = "51f12438f19735cc22c101196738cb41084c214da78793e99ea30c61a3815ebd"
diff --git a/poetry.toml b/poetry.toml
index 9255596..9d82d94 100644
--- a/poetry.toml
+++ b/poetry.toml
@@ -1,5 +1,8 @@
[experimental]
new-installer = false
+[solver]
+min-release-age = 10
+
[virtualenvs]
in-project = true
diff --git a/pyproject.toml b/pyproject.toml
index 64e4bc8..7de0cbb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,8 +31,8 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.10,<3.13" # breaking changes beyond 3.12
-boto3 = ">=1.34.162,<1.36" # breaking change beyond 1.36
-botocore = ">=1.34.162,<1.36" # breaking change beyond 1.36
+boto3 = "1.35.99" # breaking change beyond 1.36
+botocore = "1.35.99" # breaking change beyond 1.36
delta-spark = "3.2.0"
duckdb = "1.1.3" # breaking changes beyond 1.1
Jinja2 = "3.1.6"
From dd7236f5c4225cb8cb7d885dda944ab378734497 Mon Sep 17 00:00:00 2001
From: "george.robertson1" <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 22 Jun 2026 13:17:40 +0100
Subject: [PATCH 03/17] fix: adjusted the `is_type_complex` function to work
with subclasses of 'simple' types
added some more tests around is complex and stringify model to ensure functionality is working
---
src/dve/core_engine/backends/utilities.py | 8 ++-
.../test_backends/test_utilities.py | 63 ++++++++++++++++++-
2 files changed, 65 insertions(+), 6 deletions(-)
diff --git a/src/dve/core_engine/backends/utilities.py b/src/dve/core_engine/backends/utilities.py
index 6b8c5bc..f221212 100644
--- a/src/dve/core_engine/backends/utilities.py
+++ b/src/dve/core_engine/backends/utilities.py
@@ -1,5 +1,6 @@
"""Necessary, otherwise uncategorised backend functionality."""
+import inspect
import sys
from dataclasses import is_dataclass
from datetime import date, datetime, time
@@ -41,10 +42,11 @@
def is_type_complex(type_: Any) -> bool:
"""Check whether a type is a complex type or not."""
- if type_ in (str, int, float, bool, bytes):
- return False
+ _simple_types = (str, int, float, bool, bytes, date, datetime, time, Decimal)
- if type_ in (date, datetime, time, Decimal):
+ if type_ in _simple_types or (
+ inspect.isclass(type_) and any(issubclass(type_, st) for st in _simple_types)
+ ):
return False
return True
diff --git a/tests/test_core_engine/test_backends/test_utilities.py b/tests/test_core_engine/test_backends/test_utilities.py
index 839d9b7..1c9121f 100644
--- a/tests/test_core_engine/test_backends/test_utilities.py
+++ b/tests/test_core_engine/test_backends/test_utilities.py
@@ -1,10 +1,29 @@
-from datetime import date
+"""
+Test backend utility functions
+"""
+
+# pylint: disable=C0115,C0116
+
+from datetime import date, datetime
+from decimal import Decimal
from typing import Any
import pytest
from pydantic import BaseModel
-from dve.core_engine.backends.utilities import is_field_complex
+from dve.core_engine.backends.utilities import is_field_complex, stringify_model
+from dve.metadata_parser.domain_types import FormattedDatetime
+
+
+def model_to_schema(mdl: BaseModel) -> dict[str, Any]:
+ schema = {}
+ for field_name, field_md in mdl.model_fields.items():
+ if issubclass(field_md.annotation, BaseModel):
+ schema[field_name] = model_to_schema(field_md.annotation)
+ else:
+ schema[field_name] = field_md.annotation
+
+ return schema
class AnotherTestModel(BaseModel):
@@ -21,6 +40,23 @@ class TestModel(BaseModel):
simple_tuple: tuple[int, str]
another_model: AnotherTestModel
date_example: date
+ test_fdatetime: FormattedDatetime
+
+
+class ChildMultiTypeModel(BaseModel):
+ child_test_int: int
+
+
+class MultiTypeModel(BaseModel):
+ test_str: str
+ test_int: int
+ test_dict: dict[str, int]
+ test_list: list[int]
+ test_decimal: Decimal
+ test_date: date
+ test_datetime: datetime
+ test_fdatetime: FormattedDatetime
+ test_child_model: ChildMultiTypeModel
TEST_MODEL = TestModel(
@@ -32,7 +68,8 @@ class TestModel(BaseModel):
simple_set={"a", "b", "c"},
simple_tuple=(1, "wow"),
another_model=AnotherTestModel(test_field="lemon"),
- date_example=date(2026,1,1)
+ date_example=date(2026,1,1),
+ test_fdatetime="2026-01-01T12:30:10",
)
@@ -47,9 +84,29 @@ class TestModel(BaseModel):
("simple_set", True),
("simple_tuple", True),
("another_model", True),
+ ("test_fdatetime", False),
]
)
def test_is_field_complex(field_name: str, expected: bool):
_field = TEST_MODEL.model_fields[field_name]
_res = is_field_complex(_field)
assert _res == expected, f"Expected {expected}. Got {_res}."
+
+
+def test_stringify_model():
+ expected_dict = {
+ "test_str": str,
+ "test_int": str,
+ "test_dict": dict[str, str],
+ "test_list": list[str],
+ "test_decimal": str,
+ "test_date": str,
+ "test_datetime": str,
+ "test_fdatetime": str,
+ "test_child_model": {
+ "child_test_int": str
+ },
+ }
+ serialised_model_result = model_to_schema(stringify_model(MultiTypeModel))
+
+ assert expected_dict == serialised_model_result
From 4d6b0825f9a2ed81c743a16e083e9104faee841e Mon Sep 17 00:00:00 2001
From: "george.robertson1" <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 22 Jun 2026 17:33:48 +0100
Subject: [PATCH 04/17] fix: inherit is_mandatory with get_type_and_validators
This was causing custom type objects in the dischema to become unmandatory despite being lists in the mandatory_fields section
---
src/dve/metadata_parser/models.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/dve/metadata_parser/models.py b/src/dve/metadata_parser/models.py
index fd8ed6f..49e2386 100644
--- a/src/dve/metadata_parser/models.py
+++ b/src/dve/metadata_parser/models.py
@@ -231,7 +231,7 @@ def get_type_and_validators(
elif hasattr(possible_python_type, "get_type_and_validators"):
possible_python_type: "FieldSpecification" # type: ignore
nested_vals = possible_python_type.get_type_and_validators( # type: ignore
- field_name, *type_mappings, schemas=schemas, is_mandatory=False
+ field_name, *type_mappings, schemas=schemas, is_mandatory=is_mandatory
)
python_type, nested_default, nested_validators = nested_vals # type: ignore
From 6254044727a43465d1d2cefdc250230db04aca71 Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Wed, 15 Jul 2026 16:54:57 +0100
Subject: [PATCH 05/17] style: sort linting issues
---
.../backends/implementations/spark/contract.py | 7 ++++---
.../implementations/spark/spark_helpers.py | 2 +-
src/dve/core_engine/configuration/v1/__init__.py | 4 ++--
src/dve/core_engine/engine.py | 16 ++++++++--------
4 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/src/dve/core_engine/backends/implementations/spark/contract.py b/src/dve/core_engine/backends/implementations/spark/contract.py
index fe6b83e..d2fd9ae 100644
--- a/src/dve/core_engine/backends/implementations/spark/contract.py
+++ b/src/dve/core_engine/backends/implementations/spark/contract.py
@@ -104,7 +104,9 @@ def apply_data_contract(
successful = True
for entity_name, record_df in entities.items():
- entity_fields: dict[str, FieldInfo] = contract_metadata.schemas[entity_name].model_fields
+ entity_fields: dict[str, FieldInfo] = contract_metadata.schemas[
+ entity_name
+ ].model_fields
spark_schema = get_type_from_annotation(contract_metadata.schemas[entity_name])
spark_schema.add(StructField(RECORD_INDEX_COLUMN_NAME, LongType()))
if df_is_empty(record_df):
@@ -155,8 +157,7 @@ def apply_data_contract(
).alias(fld)
if fld in record_df.columns
else lit(None).cast(
- get_type_from_annotation(fld_info.annotation).alias(fld)
- )
+ get_type_from_annotation(fld_info.annotation)).alias(fld)
)
for fld, fld_info in entity_fields.items()
],
diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
index 309ddc8..981cf13 100644
--- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
+++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
@@ -12,8 +12,8 @@
from dataclasses import dataclass, is_dataclass
from decimal import Decimal
from functools import wraps
-from typing import Any, ClassVar, Literal, Optional, TypeVar, Union, overload
from pathlib import Path
+from typing import Any, ClassVar, Literal, Optional, TypeVar, Union, overload
from delta.exceptions import ConcurrentAppendException, DeltaConcurrentModificationException
from pydantic import BaseModel
diff --git a/src/dve/core_engine/configuration/v1/__init__.py b/src/dve/core_engine/configuration/v1/__init__.py
index a800647..959596f 100644
--- a/src/dve/core_engine/configuration/v1/__init__.py
+++ b/src/dve/core_engine/configuration/v1/__init__.py
@@ -282,8 +282,8 @@ def _load_rules_and_vars(self) -> tuple[list[Rule], list[TemplateVariables]]:
added_rules: set[RuleName] = set()
for index, complex_rule_config in enumerate(
- self.transformations.complex_rules
- ): # pylint: disable=E1101
+ self.transformations.complex_rules # pylint: disable=E1101
+ ):
rule, local_params, deps = self._resolve_business_rule(complex_rule_config)
missing_rules = deps - added_rules
if missing_rules:
diff --git a/src/dve/core_engine/engine.py b/src/dve/core_engine/engine.py
index 8a69a10..a7931ad 100644
--- a/src/dve/core_engine/engine.py
+++ b/src/dve/core_engine/engine.py
@@ -178,9 +178,9 @@ def __exit__(
exc_value: Optional[Exception],
traceback: Optional[TracebackType],
) -> None:
- self.main_log.info(
+ self.main_log.info( # pylint: disable=E1101
f"Exiting pipeline context, clearing {self.cache_prefix!r}"
- ) # pylint: disable=E1101
+ )
cache_dir = self._cache_dir
self._cache_dir = None
@@ -208,9 +208,9 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities:
"""
output_entities = {}
- self.main_log.info(
+ self.main_log.info( # pylint: disable=E1101
f"Writing entities to the output location: {self.output_prefix_uri}"
- ) # pylint: disable=E1101
+ )
for entity_name, entity in entities.items():
entity = entity.drop(RECORD_INDEX_COLUMN_NAME)
@@ -218,13 +218,13 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities:
output_uri = joinuri(self.output_prefix_uri, entity_name)
if get_resource_exists(output_uri):
- self.main_log.info(
+ self.main_log.info( # pylint: disable=E1101
f"{output_uri} already exists - will be overwritten"
- ) # pylint: disable=E1101
+ )
- self.main_log.info(
+ self.main_log.info( # pylint: disable=E1101
f"+ Writing parquet output to {output_uri!r}"
- ) # pylint: disable=E1101
+ )
entity.write.mode("overwrite").parquet(output_uri)
spark_session = SparkSession.builder.getOrCreate()
output_entities[entity_name] = spark_session.read.format("parquet").load(
From 08160282b01e28fad9359add04485daa98f3d07f Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Thu, 16 Jul 2026 15:37:50 +0100
Subject: [PATCH 06/17] =?UTF-8?q?bump:=20version=200.8.3=20=E2=86=92=200.9?=
=?UTF-8?q?.0rc0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
CHANGELOG.md | 11 +++++++++++
pyproject.toml | 2 +-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 690c385..630e44f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,14 @@
+## v0.9.0 (2026-07-16)
+
+### Fix
+
+- inherit is_mandatory with get_type_and_validators
+- adjusted the `is_type_complex` function to work with subclasses of 'simple' types
+
+### Refactor
+
+- upgrade pydantic to v2 (#107)
+
## v0.8.3 (2026-07-14)
### Fix
diff --git a/pyproject.toml b/pyproject.toml
index 0448e72..f8ae030 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
[tool.poetry]
-version = "0.8.3"
+version = "0.9.0rc0"
packages = [
{ include = "dve", from = "src" },
]
From 55643c01ae3acc25e573738884817b71fa91c04b Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 17 Jul 2026 14:04:11 +0100
Subject: [PATCH 07/17] docs: clarify supported os
---
pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pyproject.toml b/pyproject.toml
index f8ae030..8e08264 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,7 +11,7 @@ classifiers = [
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
- "Operating System :: OS Independent",
+ "Operating System :: POSIX :: Linux",
"Topic :: Software Development :: Libraries",
"Typing :: Typed",
]
From 4016efe255b14638744bca49e21e7a1cf38cd3e6 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 17 Jul 2026 14:31:09 +0100
Subject: [PATCH 08/17] docs: update readme for v0.9
---
README.md | 49 +++++++++++++------------------------------------
1 file changed, 13 insertions(+), 36 deletions(-)
diff --git a/README.md b/README.md
index 7c16bb5..7f1b5fc 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
- 
+ 
Data Validation Engine
@@ -9,49 +9,26 @@
[](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml)
[](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml)
-The Data Validation Engine (DVE) is a configuration driven data validation library built and utilised by NHS England. Currently the package has been reverted from v1.0.0 release to a 0.x as we feel the package is not yet mature enough to be considered a 1.0.0 release. So please bear this in mind if reading through the commits and references to a v1+ release when on v0.x.
+The Data Validation Engine (DVE) is a configuration-driven data validation library created and used by NHS England. It lets users define validation rules once and apply them across multiple dataset collections - supporting consistent, accurate data checks.
-As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` file and example files can be accessed [here](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here](https://nhsdigital.github.io/data-validation-engine/).
+__The DVE offers__:
-Once a dischema file has been defined, you are ready to use the DVE. The DVE is typically orchestrated based on four key "services". These are...
+- SQL configuration-based validations
+- Format normalization to Parquet for a unified data representation
+- Data modelling and typecasting
+- Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Clear validation reporting, including summary insights and record-level error messages
-| | Service | Purpose |
-| -- | ------- | ------- |
-| 1. | File Transformation | This service will take submitted files and turn them into stringified parquet file(s) to ensure that a consistent data structure can be passed through the other services. |
-| 2. | Data Contract | This service will validate and perform type casting against a stringified parquet file using [pydantic models](https://docs.pydantic.dev/1.10/). |
-| 3. | Business Rules | The business rules service will perform more complex validations such as comparisons between fields and tables, aggregations, filters etc to generate new entities. |
-| 4. | Error Reports | The error reports service will take all the errors raised in previous services and surface them into a readable format for a downstream users/service. Currently, this implemented to be an excel spreadsheet but could be reconfigured to meet other requirements/use cases. |
-
-If you'd like more detailed documentation around these services the please read the extended documentation [here](https://nhsdigital.github.io/data-validation-engine/).
-
-The DVE has been designed in a way that's modular and can support users who just want to utilise specific "services" from the DVE (i.e. just the file transformation + data contract). Additionally, the DVE is designed to support different backend implementations. As part of the base installation of DVE, you will find backend support for `Spark` and `DuckDB`. So, if you need a `MySQL` backend implementation, you can implement this yourself. Given our organisations requirements, it will be unlikely that we add anymore specific backend implementations into the base package beyond Spark and DuckDB. So, if you are unable to implement this yourself, I would recommend reading the guidance on [requesting new features and raising bug reports here](#requesting-new-features-and-raising-bug-reports).
-
-Additionally, if you'd like to contribute a new backend implementation into the base DVE package, then please look at the [Contributing](#Contributing) section.
+As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` (data ingest schema) file and example files can be accessed [here ↗️](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/).
## Installation and usage
-The DVE is a Python package and can be installed using package managers such as [pip](https://pypi.org/project/pip/). As of the latest release we support Python 3.10 & 3.11, with Spark v3.4 and DuckDB v1.1. In the future we will be looking to upgrade the DVE to working on a higher versions of Python, DuckDB and Spark.
-
-If you're planning to use the Spark backend implementation, you will also need OpenJDK 11 installed.
-
-Python dependencies are listed in `pyproject.toml`.
-
-To install the DVE package you can simply install using a package manager such as [pip](https://pypi.org/project/pip/).
-
-```
-pip install data-validation-engine
-```
-
-*Note - Only versions >=0.6.2 are available on PyPi. For older versions please install directly from the git repo or build from source.*
-
-Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema JSON document (configuration), please read the [documentation](https://nhsdigital.github.io/data-validation-engine/).
-
-Version 0.0.1 does support a working Python 3.7 installation. However, we will not be supporting any issues with that version of the DVE if you choose to use it. __Use at your own risk__.
+Please see the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/user_guidance/install/).
## Requesting new features and raising bug reports
**Before creating new issues, please check to see if the same bug/feature has been created already. Where a duplicate is created, the ticket will be closed and referenced to an existing issue.**
-If you have spotted a bug with the DVE then please raise an issue [here](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template".
+If you have spotted a bug with the DVE then please raise an issue [here ↗️](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template".
If you have feature request then please follow the same process whilst using the "Feature request template".
@@ -63,7 +40,7 @@ Below is a list of features that we would like to implement or have been request
| Uplift to Python 3.11 | 0.2.0 | Yes |
| Uplift Pyspark to 3.5 | 0.8.0 | Yes |
| Allow DVE to run on Python 3.12+ | 0.8.0 | Yes |
-| Upgrade to Pydantic 2.0 | 0.9.0 | No |
+| Upgrade to Pydantic 2.0 | 0.9.0 | Yes |
| Uplift Pyspark to 4.0+ | TBA | No |
| Polars upgrade to v1+ | TBA | No |
| DuckDB upgrade to v1.5+ | TBA | No |
@@ -73,7 +50,7 @@ Below is a list of features that we would like to implement or have been request
If you are interested in getting any of the unreleased features listed above available, then please read the [Contributing](#Contributing) section and then submit us a pull request.
## Contributing
-Please see guidance [here](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md).
+Please see guidance [here ↗️](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md).
## Legal
This codebase is released under the MIT License. This covers both the codebase and any sample code in the documentation.
From 1f6922d9010a2fb3fa85d8ea6ec8e84208eb31bc Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 17 Jul 2026 14:38:30 +0100
Subject: [PATCH 09/17] docs: add security.md
---
SECURITY.md | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100644 SECURITY.md
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..abb9914
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,23 @@
+# Security
+
+We take security and the protection of private data extremely seriously. If you believe you have found a vulnerability or other issue which has compromised or could compromise the security of any of our systems or private data managed by our systems, please do not hesitate to contact us using the method outlined below.
+
+## Table of contents
+
+- [Security](#security)
+- [Table of contents](#table-of-contents)
+- [Reporting a vulnerability](#reporting-a-vulnerability)
+- [General security enquiries](#general-security-enquiries)
+
+## Reporting a vulnerability
+
+If you believe you have found a security issue in this repository, please report it using GitHub's private vulnerability reporting:
+
+1. [Report a vulnerability](https://github.com/NHSDigital/data-validation-engine/security/advisories/new)
+2. Provide details of the issue and steps to reproduce
+
+This creates a private channel for discussion and allows us to coordinate a fix before any public disclosure.
+
+## General security enquiries
+
+If you have general enquiries regarding our cybersecurity, please reach out to us at [cybersecurity@nhs.net](cybersecurity@nhs.net)
From 809ec016128ec209a99d5dc2229a2c65bec686cf Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 17 Jul 2026 14:40:09 +0100
Subject: [PATCH 10/17] docs: rename CONTRIBUTE.md
---
CONTRIBUTE.md => CONTRIBUTING.md | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename CONTRIBUTE.md => CONTRIBUTING.md (100%)
diff --git a/CONTRIBUTE.md b/CONTRIBUTING.md
similarity index 100%
rename from CONTRIBUTE.md
rename to CONTRIBUTING.md
From df952f4864bf07383d4b701363445ebf1c4a746d Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 17 Jul 2026 17:19:15 +0100
Subject: [PATCH 11/17] build: update docs around references to pydantic v1
---
docs/user_guidance/data_contract.md | 18 +++++++-----------
docs/user_guidance/install.md | 25 ++++++++++---------------
2 files changed, 17 insertions(+), 26 deletions(-)
diff --git a/docs/user_guidance/data_contract.md b/docs/user_guidance/data_contract.md
index 55913da..13e72ef 100644
--- a/docs/user_guidance/data_contract.md
+++ b/docs/user_guidance/data_contract.md
@@ -6,11 +6,7 @@ tags:
- Domain Types
---
-The Data Contract defines the structure (models) of your data and controls how it is typecast. We use [Pydantic](https://pydantic.dev/docs/validation/1.10/overview/) to generate and validate the models. This page is meant to give you greater details on how you should write your Data Contract. If you want a summary of how the Data Contract works, please refer to the [Getting Started](./getting_started.md#rules-configuration-introduction) page.
-
-!!! Note
-
- We plan to migrate to Pydantic v2+ in v0.9.0. This page currently reflects what is available through Pydantic v1.
+The Data Contract defines the structure (models) of your data and controls how it is typecast. We use [Pydantic v2](https://pydantic.dev/docs/validation/latest/get-started/) to generate and validate the models. This page is meant to give you greater details on how you should write your Data Contract. If you want a summary of how the Data Contract works, please refer to the [Getting Started](./getting_started.md#rules-configuration-introduction) page.
## Models
@@ -207,15 +203,15 @@ If you want to read more about the readers, please see the [File Transformation]
Within the `fields` section of the contract you must define what data type a given field should be. Depending on how strict/lenient you want your types to be, a number of types are available to use. The types available are:
- [Built-in standard library](https://docs.python.org/3.12/library/stdtypes.html) types (such as `int`, `str`, `date`) available with your version of Python installed for the DVE.
-- [Pydantic v1 types](https://docs.pydantic.dev/1.10/usage/types/)
+- [Pydantic v2 types](https://pydantic.dev/docs/validation/latest/concepts/types/)
- [Custom Types](./data_contract.md#custom-types)
- [Domain types](./data_contract.md#domain-types)
### Constraints
-Given the DVE supports Pydantic types, you can use any of the [constrained types available](https://docs.pydantic.dev/1.10/usage/types/#constrained-types). The Pydantic docs will also show you what `kwarg` arguments are available for each constraint such as min/max length, regex patterns etc.
+The DVE supports constraints against valid python/pydantic types. You can use the `constraints` arguments to apply a specifc constraint such as min/max length etc.
-For example, if you wanted to use a `constr` type for a field, you would define it like this:
+For example, if you wanted to use apply constraints to a `str` type, you may define it like this:
```json title="movies.dischema.json"
{
@@ -225,7 +221,7 @@ For example, if you wanted to use a `constr` type for a field, you would define
"fields": {
"movie_uuid": "int",
"movie_name": {
- "callable": "constr",
+ "callable": "str",
"constraints": {
"min_length": 1,
"max_length": 20
@@ -249,7 +245,7 @@ As shown in the [Constraints](./data_contract.md#constraints) section above, you
{
"types": {
"MyConstrainedString": {
- "callable": "constr",
+ "callable": "str",
"constraints": {
"min_length": 1,
"max_length": 20
@@ -334,7 +330,7 @@ To define a struct type, you would add it to the `types` section like this...
"fields": {
"movie_uuid": "int",
"movie_name": {
- "callable": "constr",
+ "callable": "str",
"constraints": {
"min_length": 1,
"max_length": 20
diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md
index e0c2ce4..893b229 100644
--- a/docs/user_guidance/install.md
+++ b/docs/user_guidance/install.md
@@ -8,11 +8,11 @@ tags:
!!! warning
**DVE is currently an unstable package. Expect breaking changes between every minor patch**. We intend to follow semantic versioning of `major.minor.patch` more strictly after a 1.0 release. Until then, we recommend that you pin your install to the latest version available and keep an eye on [future releases](https://github.com/NHSDigital/data-validation-engine/releases).
- **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.12, but it's not an immediate priority.
+ **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.13, but it's not an immediate priority.
If working on Python 3.7, the `0.1` release supports this (and only this) version of Python. However, we have not been updating that version with any bugfixes, performance improvements etc. There are also a number of vulnerable dependencies on version `0.1` release due to [Python 3.7 being depreciated](https://devguide.python.org/versions/) and a number of packages dropping support. **If you choose to install `0.1`, you accept the risks of doing so and additional support will not be provided.**
-You can install the DVE package through python package managers such as [pip](https://pypi.org/project/pip/), [pipx](https://github.com/pypa/pipx), [uv](https://docs.astral.sh/uv/) and [poetry](https://python-poetry.org/).
+You can install the DVE package through python package managers such as [pip](https://pypi.org/project/pip/), [uv](https://docs.astral.sh/uv/) and [poetry](https://python-poetry.org/).
=== "pip"
@@ -20,12 +20,6 @@ You can install the DVE package through python package managers such as [pip](ht
pip install data-validation-engine
```
-=== "pipx"
-
- ```sh
- pipx install data-validation-engine
- ```
-
=== "uv"
Add to your existing `uv` project...
@@ -84,10 +78,11 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru
## DVE Version Compatability Matrix
-| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version |
-| ------------ | -------------- | -------------- | ------------- | ---------------- |
-| >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 |
-| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 |
-| >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 |
-| >=0.2,<0.6 | >=3.10,<3.12 | 1.1.0 | 3.4.4 | 1.10.15 |
-| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 |
+| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version |
+| ------------ | -------------- | -------------- | --------------- | ---------------- |
+| >=0.9.0 | >=3.10,<3.12 | 1.1.3 | >=3.5.0,<=3.5.5 | 2.13.4 |
+| >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 |
+| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 |
+| >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 |
+| >=0.2,<0.6 | >=3.10,<3.12 | 1.1.0 | 3.4.4 | 1.10.15 |
+| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 |
From 55d5e040ce79bab1e2cf7b2de48eb82cefa70ea0 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 20 Jul 2026 09:58:16 +0100
Subject: [PATCH 12/17] docs: clarify python version
---
docs/user_guidance/install.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md
index 893b229..5ad4d50 100644
--- a/docs/user_guidance/install.md
+++ b/docs/user_guidance/install.md
@@ -8,7 +8,7 @@ tags:
!!! warning
**DVE is currently an unstable package. Expect breaking changes between every minor patch**. We intend to follow semantic versioning of `major.minor.patch` more strictly after a 1.0 release. Until then, we recommend that you pin your install to the latest version available and keep an eye on [future releases](https://github.com/NHSDigital/data-validation-engine/releases).
- **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.13, but it's not an immediate priority.
+ **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.12, but it's not an immediate priority.
If working on Python 3.7, the `0.1` release supports this (and only this) version of Python. However, we have not been updating that version with any bugfixes, performance improvements etc. There are also a number of vulnerable dependencies on version `0.1` release due to [Python 3.7 being depreciated](https://devguide.python.org/versions/) and a number of packages dropping support. **If you choose to install `0.1`, you accept the risks of doing so and additional support will not be provided.**
From a5402765a0418317d8a44ae44a4926857e704011 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 20 Jul 2026 11:38:54 +0100
Subject: [PATCH 13/17] docs: add faq page
---
docs/faq.md | 24 ++++++++++++++++++++++++
zensical.toml | 3 ++-
2 files changed, 26 insertions(+), 1 deletion(-)
create mode 100644 docs/faq.md
diff --git a/docs/faq.md b/docs/faq.md
new file mode 100644
index 0000000..af25fbe
--- /dev/null
+++ b/docs/faq.md
@@ -0,0 +1,24 @@
+---
+title: Frequently Asked Questions (FAQ)
+tags:
+ - FAQ
+ - Business Rules
+ - Error Reports
+---
+
+??? question "The `id` field in the error report is missing when it should be present?"
+
+ If the `id` field isn't populated then it will be due to the fact that the key field is not present in the entity you are looking up against. The `key_fields`/`key_fields` is sourced from a dictionary lookup sourced in the data contract, so new entities defined within the business_rules won't have this lookup and it will result in the column not being populated. To get around this issue for entities without a defined `key_field` you can use the `original_entity_override` key to define the entity you want to source your `key_field`/`key_fields` from. You may find an example below.
+
+ ??? example
+
+ ```json
+ "filters": [
+ {
+ "entity": "entity_not_defined_in_data_contract",
+ "name": "my rule",
+ "expression": "a < b",
+ "original_entity_override": "entity_defined_in_data_contract"
+ }
+ ]
+ ```
diff --git a/zensical.toml b/zensical.toml
index 255a771..064cb26 100644
--- a/zensical.toml
+++ b/zensical.toml
@@ -67,7 +67,8 @@ nav = [
{"Implementing a new backend" = "advanced_guidance/new_backend.md"},
{"Dischema Language Server" = "advanced_guidance/json_schemas.md"},
]},
- ]}
+ ]},
+ {"FAQ" = "faq.md"}
]
extra_css = ["assets/stylesheets/extra.css"]
# extra_javascript = ["assets/javascript/extra.js"]
From eb6417f53ac327cb854687ec9b65d80d98819946 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 20 Jul 2026 12:01:57 +0100
Subject: [PATCH 14/17] docs: fix python version constraint for v0.9.0 release
---
docs/user_guidance/install.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md
index 5ad4d50..85186cd 100644
--- a/docs/user_guidance/install.md
+++ b/docs/user_guidance/install.md
@@ -80,7 +80,7 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru
| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version |
| ------------ | -------------- | -------------- | --------------- | ---------------- |
-| >=0.9.0 | >=3.10,<3.12 | 1.1.3 | >=3.5.0,<=3.5.5 | 2.13.4 |
+| >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | 2.13.4 |
| >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 |
| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 |
| >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 |
From 54234647c639efec824e9e73468e461b85bd51b4 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Tue, 21 Jul 2026 18:19:20 +0100
Subject: [PATCH 15/17] docs(jsonschema): add error_details json schema and
docs around them
---
docs/advanced_guidance/json_schemas.md | 6 +++
.../contract/contract.schema.json | 4 ++
.../contract/error_details.schema.json | 53 +++++++++++++++++++
3 files changed, 63 insertions(+)
create mode 100644 docs/advanced_guidance/json_schemas/contract/error_details.schema.json
diff --git a/docs/advanced_guidance/json_schemas.md b/docs/advanced_guidance/json_schemas.md
index 5f24bd4..1c65088 100644
--- a/docs/advanced_guidance/json_schemas.md
+++ b/docs/advanced_guidance/json_schemas.md
@@ -17,6 +17,12 @@ For autocomplete support in VS Code, you can alter the `.vscode/settings.json` a
],
"url": "./json_schemas/dataset.schema.json"
},
+ {
+ "fileMatch": [
+ "*_data_contract_error_details.json"
+ ],
+ "url": "./json_schemas/contract/error_details.schema.json"
+ },
{
"fileMatch": [
"*.rulestore.json",
diff --git a/docs/advanced_guidance/json_schemas/contract/contract.schema.json b/docs/advanced_guidance/json_schemas/contract/contract.schema.json
index 97c6162..dad5a92 100644
--- a/docs/advanced_guidance/json_schemas/contract/contract.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/contract.schema.json
@@ -25,6 +25,10 @@
"additionalProperties": {
"$ref": "components/readable_entity.schema.json"
}
+ },
+ "error_details": {
+ "description": "Relative path from the dischema document to reference a error details document.",
+ "type": "string"
}
},
"additionalProperties": true
diff --git a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json
new file mode 100644
index 0000000..58c369b
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json
@@ -0,0 +1,53 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "data-ingest:error_details.schema.json",
+ "title": "Data Contract Error Details",
+ "type": "object",
+ "patternProperties": {
+ ".*": {
+ "type": "object",
+ "description": "Name of the field defined within the data contract entities.",
+ "properties": {
+ "Bad value": {
+ "$ref": "#/$defs/errorObject",
+ "description": "When a value is incompatiable. e.g. trying to cast 'hello' as int."
+ },
+ "Blank": {
+ "$ref": "#/$defs/errorObject",
+ "description": "When a mandatory value is missing."
+ }
+ }
+ }
+ },
+ "$defs": {
+ "errorObject": {
+ "type": "object",
+ "properties": {
+ "error_code": {
+ "type": "string",
+ "description": "The code you want to report in the error report."
+ },
+ "error_message": {
+ "type": "string",
+ "description": "The error message you want to report in the error report."
+ },
+ "error_level": {
+ "type": "string",
+ "description": "Whether the error should be a record rejection or file/submission rejection. Defaulted to record.",
+ "enum": [
+ "record",
+ "submission"
+ ]
+ },
+ "is_informational": {
+ "type": "boolean",
+ "description": "Whether to raise the error as a warning or not. Defaulted to false."
+ },
+ "reporting_entity": {
+ "type": "string",
+ "description": "Whether to rename the entity shown in the error report. Defaults to the entity name in the dischema."
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
From 29e1ac75ca66a0075130be65fcd2493d955a04f3 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Tue, 4 Aug 2026 09:47:40 +0100
Subject: [PATCH 16/17] docs: add extra info on readme regarding functionality
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 7f1b5fc..e94b29e 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,7 @@ __The DVE offers__:
- Format normalization to Parquet for a unified data representation
- Data modelling and typecasting
- Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Deriving new fields and entities
- Clear validation reporting, including summary insights and record-level error messages
As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` (data ingest schema) file and example files can be accessed [here ↗️](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/).
From 4fdf21bbaa9d4113de78b1cea164d1a428edaaf8 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Tue, 4 Aug 2026 10:02:41 +0100
Subject: [PATCH 17/17] docs: refactored and updated the json schemas (#131)
---
.../components/base_entity.schema.json | 9 +-
.../contact_error_details.schema.json | 2 +-
.../contract/components/entity.schema.json | 2 +-
.../contract/components/field.schema.json | 6 +-
.../components/field_error_detail.schema.json | 2 +-
.../field_error_type.schema copy.json | 21 --
.../components/field_error_type.schema.json | 2 +-
.../field_specification.schema.json | 256 +++++++++++++-----
.../components/readable_entity.schema.json | 154 +++++++++--
.../ddb_csv_header_reader.schema.json | 22 ++
.../ddb_csv_reader.schema.json | 26 ++
.../ddb_json_reader.schema.json | 19 ++
.../ddb_xml_reader.schema.json | 12 +
.../global_csv_reader_args.schema.json | 25 ++
.../global_xml_reader_args.schema.json | 59 ++++
.../polars_ddb_csv_reader.schema.json | 26 ++
.../spark_csv_reader.schema.json | 38 +++
.../spark_json_reader.schema.json | 17 ++
.../spark_xml_reader.schema.json | 43 +++
.../spark_xml_stream_reader.schema.json | 12 +
.../type_constraints/alphanumeric.schema.json | 17 ++
.../type_constraints/condate.schema.json | 25 ++
.../type_constraints/condecimal.schema.json | 37 +++
.../conformatteddate.schema.json | 34 +++
.../type_constraints/conint.schema.json | 33 +++
.../type_constraints/constr.schema.json | 33 +++
.../formatteddatetime.schema.json | 21 ++
.../formattedtime.schema.json | 21 ++
.../type_constraints/identifier.schema.json | 12 +
.../type_constraints/nhsnumber.schema.json | 7 +
.../type_constraints/orgid.schema.json | 17 ++
.../permissivenhsno.schema.json | 19 ++
.../type_constraints/postcode.schema.json | 22 ++
.../reportingperiodend.schema.json | 22 ++
.../reportingperiodstart.schema.json | 22 ++
.../contract/components/type_name.schema.json | 28 +-
.../validation_function.schema.json | 34 ---
.../contract/contract.schema.json | 13 +-
.../contract/error_details.schema.json | 1 +
.../json_schemas/dataset.schema.json | 2 +-
.../json_schemas/rule_store.schema.json | 2 +-
.../components/business_filter.schema.json | 24 +-
.../components/business_rule.schema.json | 38 ++-
.../components/concrete_filter.schema.json | 2 +-
.../components/core_filter.schema.json | 46 +++-
.../components/filter.schema.json | 2 +-
.../multiple_expressions.schema.json | 2 +-
.../components/rule.schema.json | 2 +-
.../transformations.schema.json | 54 +++-
49 files changed, 1112 insertions(+), 233 deletions(-)
delete mode 100644 docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json
create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json
delete mode 100644 docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json
diff --git a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json
index cff1d6d..b89291a 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/base_entity.schema.json",
"title": "base_entity",
"description": "A complete entity specification within data ingest",
@@ -10,7 +10,8 @@
"description": "A mapping of field names to their Python types. These will either be strings representing Python types (if there are no argumements to the type), and field specification objects otherwise",
"additionalProperties": {
"$ref": "field.schema.json"
- }
+ },
+ "minProperties": 1
},
"aliases": {
"description": "A mapping of field name to allowed field alias",
@@ -26,5 +27,7 @@
"type": "string"
}
}
- }
+ },
+
+ "required": ["fields"]
}
diff --git a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json
index 2635ac4..a9a6a91 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/contract_error_details.schema.json",
"title": "base_entity",
"description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)",
diff --git a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json
index 24453fb..b751824 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/entity.schema.json",
"title": "entity",
"description": "A concrete entity specification within data ingest",
diff --git a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json
index c8454ab..f0e7d20 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/field.schema.json",
"title": "field",
"description": "The type of a field, specified as a bare Python type if there are no argumements, and as an object otherwise. This can also be a reference to a type defined in the dataset",
@@ -9,6 +9,10 @@
},
{
"$ref": "field_specification.schema.json"
+ },
+ {
+ "type": "string",
+ "minLength": 1
}
]
}
diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json
index 5f00ca9..1939d29 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/field_error_detail.schema.json",
"title": "field_error_detail",
"description": "The custom details to be used for a field when a validation error is raised during the data contract phase",
diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json
deleted file mode 100644
index 694948a..0000000
--- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$schema": "https://json-schema.org/draft-07/schema",
- "$id": "data-ingest:contract/components/field_error_type.schema.json",
- "title": "field_error_detail",
- "description": "The error type for a field when a validation error is raised during the data contract phase",
- "type": "object",
- "properties": {
- "error_type": {
- "description": "The type of error the details are for",
- "type": "string",
- "enum": [
- "Blank",
- "Bad value",
- "Wrong format"
- ],
- "additionalProperties": {
- "$ref": "field_error_detail.schema.json"
- }
- }
- }
-}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json
index 694948a..13f1a26 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/field_error_type.schema.json",
"title": "field_error_detail",
"description": "The error type for a field when a validation error is raised during the data contract phase",
diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json
index 5a57f4c..5564e18 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json
@@ -1,16 +1,16 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/field_specification.schema.json",
"title": "field_specification",
"description": "A complete field specification within data ingest. This is generally specified using a type, model, or callable (with optional constraints) which results in a type",
"type": "object",
"properties": {
"human_readable_name": {
- "description": "A human readable name for the field",
+ "description": "A human readable name for the field. Non functional.",
"type": "string"
},
"description": {
- "description": "A human-readable description of the field's purpose",
+ "description": "A human-readable description of the field's purpose. Non functional.",
"type": "string"
},
"type": {
@@ -36,108 +36,216 @@
"default": {
"description": "A default value for the field, to be used if it is not provided"
},
- "functions": {
- "description": "Validation functions to be applied to the type",
- "type": "array",
- "items": {
- "$ref": "validation_function.schema.json"
- }
- },
"examples": {
- "description": "An array of example values that the field could contain",
+ "description": "An array of example values that the field could contain. Non functional.",
"type": "array"
},
"disallowed_examples": {
- "description": "An array of example values that the field could not contain",
+ "description": "An array of example values that the field could not contain. Non functional.",
"type": "array"
}
},
- "dependencies": {
- "type": {
- "allOf": [
+ "allOf": [
+ {
+ "oneOf": [
+ { "required": ["type"] },
+ { "required": ["model"] },
+ { "required": ["callable"] }
+ ],
+ "errorMessage": "Must specify exactly one of: type, model, or callable"
+ },
+ {
+ "not": {
+ "required": ["type", "model"]
+ },
+ "errorMessage": "Type and model are mutually exclusive"
+ },
+ {
+ "not": {
+ "required": ["type", "callable"]
+ },
+ "errorMessage": "Type and callable are mutually exclusive"
+ },
+ {
+ "not": {
+ "required": ["model", "callable"]
+ },
+ "errorMessage": "Model and callable are mutually exclusive"
+ },
+ {
+ "if": { "required": ["callable"] },
+ "then": {
+ "required": ["constraints"],
+ "errorMessage": "constraints is required when using callable"
+ }
+ }
+ ],
+ "unevaluatedProperties": false,
+ "dependentSchemas": {
+ "callable": {
+ "oneOf": [
{
- "not": {
- "required": [
- "model"
- ]
+ "properties": {
+ "callable": {
+ "const": "alphanumeric"
+ },
+ "constraints": {
+ "$ref": "type_constraints/alphanumeric.schema.json"
+ }
}
},
{
- "not": {
- "required": [
- "callable"
- ]
+ "properties": {
+ "callable": {
+ "const": "condate"
+ },
+ "constraints": {
+ "$ref": "type_constraints/condate.schema.json"
+ }
}
},
{
- "not": {
- "required": [
- "constraints"
- ]
+ "properties": {
+ "callable": {
+ "const": "condecimal"
+ },
+ "constraints": {
+ "$ref": "type_constraints/condecimal.schema.json"
+ }
}
- }
- ]
- },
- "model": {
- "allOf": [
+ },
{
- "not": {
- "required": [
- "type"
- ]
+ "properties": {
+ "callable": {
+ "const": "conformatteddate"
+ },
+ "constraints": {
+ "$ref": "type_constraints/conformatteddate.schema.json"
+ }
}
},
{
- "not": {
- "required": [
- "callable"
- ]
+ "properties": {
+ "callable": {
+ "const": "conint"
+ },
+ "constraints": {
+ "$ref": "type_constraints/conint.schema.json"
+ }
}
},
{
- "not": {
- "required": [
- "constraints"
- ]
+ "properties": {
+ "callable": {
+ "const": "constr"
+ },
+ "constraints": {
+ "$ref": "type_constraints/constr.schema.json"
+ }
}
- }
- ]
- },
- "callable": {
- "allOf": [
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "formatteddatetime"
+ },
+ "constraints": {
+ "$ref": "type_constraints/formatteddatetime.schema.json"
+ }
+ }
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "formattedtime"
+ },
+ "constraints": {
+ "$ref": "type_constraints/formattedtime.schema.json"
+ }
+ }
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "identifier"
+ },
+ "constraints": {
+ "$ref": "type_constraints/identifier.schema.json"
+ }
+ }
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "postcode"
+ },
+ "constraints": {
+ "$ref": "type_constraints/postcode.schema.json"
+ }
+ }
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "postcode"
+ },
+ "constraints": {
+ "$ref": "type_constraints/postcode.schema.json"
+ }
+ }
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "nhsnumber"
+ },
+ "constraints": {
+ "$ref": "type_constraints/nhsnumber.schema.json"
+ }
+ }
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "permissivenhsno"
+ },
+ "constraints": {
+ "$ref": "type_constraints/permissivenhsno.schema.json"
+ }
+ }
+ },
+ {
+ "properties": {
+ "callable": {
+ "const": "orgid"
+ },
+ "constraints": {
+ "$ref": "type_constraints/orgid.schema.json"
+ }
+ }
+ },
{
- "not": {
- "required": [
- "type"
- ]
+ "properties": {
+ "callable": {
+ "const": "reportingperiodstart"
+ },
+ "constraints": {
+ "$ref": "type_constraints/reportingperiodstart.schema.json"
+ }
}
},
{
- "not": {
- "required": [
- "model"
- ]
+ "properties": {
+ "callable": {
+ "const": "reportingperiodend"
+ },
+ "constraints": {
+ "$ref": "type_constraints/reportingperiodend.schema.json"
+ }
}
}
]
}
},
- "oneOf": [
- {
- "required": [
- "type"
- ]
- },
- {
- "required": [
- "model"
- ]
- },
- {
- "required": [
- "callable"
- ]
- }
- ],
"additionalProperties": false
}
diff --git a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json
index 3f0954e..fdf2565 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/readable_entity.schema.json",
"title": "entity",
"description": "A concrete entity specification within data ingest. This must be readable from file",
@@ -13,32 +13,144 @@
"reader_config": {
"description": "Reader configuration for each entity. File extensions should be used as keys.",
"type": "object",
- "patternProperties": {
- "^\\..+$": {
- "type": "object",
- "properties": {
- "reader": {
- "type": "string",
- "description": "The name of the reader class that should be used for the file extension."
- },
- "kwargs": {
- "type": "object",
- "description": "Additional keyword constructor args for the reader class."
- }
- }
+ "properties": {
+ ".csv": {
+ "$ref": "#/$defs/csvReaderConfig",
+ "description": "Configuration for loading an CSV file.",
+ "required": ["reader"]
+ },
+ ".json": {
+ "$ref": "#/$defs/jsonReaderConfig",
+ "description": "Configuration for loading an JSON file.",
+ "required": ["reader"]
+ },
+ ".xml": {
+ "$ref": "#/$defs/xmlReaderConfig",
+ "description": "Configuration for loading an XML file.",
+ "required": ["reader"]
}
},
- "additionalProperties": false
+ "minProperties": 1
},
"key_field": {
"description": "The key field that should be used for the entity.",
"type": "string"
}
- },
- "required": [
- "fields",
- "reader_config"
+ }
+ }
+ ],
+ "$defs": {
+ "csvReaderConfig": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "DuckDBCSVReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/ddb_csv_reader.schema.json"
+ }
+ }
+ },
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "PolarsToDuckDBCSVReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/polars_ddb_csv_reader.schema.json"
+ }
+ }
+ },
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "DuckDBCSVRepeatingHeaderReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/ddb_csv_header_reader.schema.json"
+ }
+ }
+ },
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "SparkCSVReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/spark_csv_reader.schema.json"
+ }
+ }
+ }
+ ]
+ },
+ "jsonReaderConfig": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "DuckDBJSONReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/ddb_json_reader.schema.json"
+ }
+ }
+ },
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "SparkJSONReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/spark_json_reader.schema.json"
+ }
+ }
+ }
+ ]
+ },
+ "xmlReaderConfig": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "DuckDBXMLReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/ddb_xml_reader.schema.json"
+ }
+ }
+ },
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "SparkXMLStreamReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/spark_xml_stream_reader.schema.json"
+ }
+ }
+ },
+ {
+ "type": "object",
+ "properties": {
+ "reader": {
+ "const": "SparkXMLReader"
+ },
+ "kwargs": {
+ "$ref": "reader_constraints/spark_xml_reader.schema.json"
+ }
+ }
+ }
]
}
- ]
-}
+ },
+ "required": ["fields", "reader_config"]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json
new file mode 100644
index 0000000..7a6cb48
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json
@@ -0,0 +1,22 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_header_reader.schema.json",
+ "title": "Keyword Arguments a DuckDBCSVRepeatingHeaderReader",
+ "description": "Arguments to contol how the DuckDB CSV Header Reader interacts with a csv file",
+ "type": "object",
+ "anyOf": [
+ {
+ "$ref": "ddb_csv_reader.schema.json"
+ }
+ ],
+ "properties": {
+ "non_unique_header_error_code": {
+ "type": "string",
+ "description": "Allows you to customise the error code generated in the error report when there are more than 1 set of values for a header."
+ },
+ "non_unique_header_error_message": {
+ "type": "string",
+ "description": "Allows you to customise the error message generated in the error report when there are more than 1 set of values for a header."
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json
new file mode 100644
index 0000000..8a531b5
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json
@@ -0,0 +1,26 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_reader.schema.json",
+ "title": "Keyword Arguments a DuckDBCSVReader",
+ "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file",
+ "type": "object",
+ "anyOf": [
+ {
+ "$ref": "global_csv_reader_args.schema.json"
+ }
+ ],
+ "properties": {
+ "header": {
+ "type": "boolean",
+ "description": "First line of each file contains the column names."
+ },
+ "delim": {
+ "type": "string",
+ "description": "Delimiter character used to separate columns within each line."
+ },
+ "quotechar": {
+ "type": "string",
+ "description": "Character used to denote the start and end of a quoted item."
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json
new file mode 100644
index 0000000..7cb4576
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json
@@ -0,0 +1,19 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/ddb_json_reader.schema.json",
+ "title": "Keyword Arguments for DuckDBJSONReader",
+ "description": "Arguments to contol how the DuckDB JSON Reader interacts with a json file",
+ "type": "object",
+ "properties": {
+ "json_format": {
+ "type": "string",
+ "description": "The format of the json file. Defaults to 'array'.",
+ "enum": [
+ "auto",
+ "unstructured",
+ "newline_delimited",
+ "array"
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json
new file mode 100644
index 0000000..4872335
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json
@@ -0,0 +1,12 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/ddb_xml_reader.schema.json",
+ "title": "Keyword Arguments for DuckDBXMLStreamReader",
+ "description": "Arguments to contol how the DuckDB XML Reader interacts with a xml file",
+ "type": "object",
+ "anyOf": [
+ {
+ "$ref": "global_xml_reader_args.schema.json"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json
new file mode 100644
index 0000000..4ea82ec
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json
@@ -0,0 +1,25 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/global_csv_reader_args.schema.json",
+ "title": "Keyword Arguments used across all CSV readers",
+ "description": "Arguments present in all CSV readers available.",
+ "type": "object",
+ "properties": {
+ "field_check": {
+ "type": "boolean",
+ "description": "Enable to check for additional or missing fields."
+ },
+ "field_check_error_code": {
+ "type": "string",
+ "description": "Allows you to customise the error code generated in the error report when the field check fails."
+ },
+ "field_check_error_message": {
+ "type": "string",
+ "description": "Allows you to customise the error message generated in the error report when the field check fails."
+ },
+ "null_empty_strings": {
+ "type": "boolean",
+ "description": "Converts empty string values into 'Null' values."
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json
new file mode 100644
index 0000000..d82ce72
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json
@@ -0,0 +1,59 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/global_xml_reader_args.schema.json",
+ "title": "Keyword Arguments used across all XML readers",
+ "description": "Arguments present in all XML readers.",
+ "type": "object",
+ "properties": {
+ "record_tag": {
+ "type": "string",
+ "description": "A required string indicating the tag of each 'record' in the XML document."
+ },
+ "root_tag": {
+ "type": "string",
+ "description": "A string indicating the tag to find the records in within the XML document."
+ },
+ "trim_cells": {
+ "type": "string",
+ "description": "Removes leading and trailing whitespaces."
+ },
+ "null_values": {
+ "type": "array",
+ "description": "A container of values to replace with null if encountered in an element.",
+ "items": {
+ "type": "string"
+ }
+ },
+ "sanitise_multiline": {
+ "type": "boolean",
+ "description": "Whether to sanitise (remove newlines and multiple spaces) from multiline fields"
+ },
+ "encoding": {
+ "type": "string",
+ "description": "Encoding of the XML file. Default: `utf-8-sig`"
+ },
+ "n_records_to_read": {
+ "type": "integer",
+ "description": "The maximum number of records to read from a document"
+ },
+ "xsd_location": {
+ "type": "string",
+ "description": "The Relative path (to the dischema) for the XSD document."
+ },
+ "xsd_error_code": {
+ "type": "string",
+ "description": "Allows you to customise the error code generated in the error report when the XSD check fails."
+ },
+ "xsd_error_message": {
+ "type": "string",
+ "description": "Allows you to customise the error message generated in the error report when the XSD check fails."
+ },
+ "rules_location": {
+ "type": "string",
+ "description": "Allows you to prefix the directory which contains the XSD document."
+ }
+ },
+ "required": [
+ "record_tag"
+ ]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json
new file mode 100644
index 0000000..7564660
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json
@@ -0,0 +1,26 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/polars_ddb_csv_reader.schema.json",
+ "title": "Keyword Arguments a DuckDBCSVReader",
+ "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file",
+ "type": "object",
+ "anyOf": [
+ {
+ "$ref": "global_csv_reader_args.schema.json"
+ }
+ ],
+ "properties": {
+ "has_header": {
+ "type": "boolean",
+ "description": "First line of each file contains the column names."
+ },
+ "separator": {
+ "type": "string",
+ "description": "Delimiter character used to separate columns within each line."
+ },
+ "quote_char": {
+ "type": "string",
+ "description": "Character used to denote the start and end of a quoted item."
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json
new file mode 100644
index 0000000..e80dd7a
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json
@@ -0,0 +1,38 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/spark_csv_reader.schema.json",
+ "title": "Keyword Arguments a SparkCSVReader",
+ "description": "Arguments to contol how the Spark CSV Reader interacts with a csv file",
+ "type": "object",
+ "anyOf": [
+ {
+ "$ref": "global_csv_reader_args.schema.json"
+ }
+ ],
+ "properties": {
+ "delimiter": {
+ "type": "string",
+ "description": "Sets a separator for each field and value. This separator can be one or more characters."
+ },
+ "escape_char": {
+ "type": "string",
+ "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quotes inside an already quoted value."
+ },
+ "quote_char": {
+ "type": "string",
+ "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quoted values where the separator can be part of the value. For reading, if you would like to turn off quotations, you need to set not null but an empty string."
+ },
+ "header": {
+ "type": "boolean",
+ "description": "For reading, uses the first line as names of columns. Note that if the given path is a RDD of Strings, this header option will remove all lines same with the header if exists. CSV built-in functions ignore this option."
+ },
+ "multi_line": {
+ "type": "boolean",
+ "description": "Allows a row to span multiple lines, by parsing line breaks within quoted values as part of the value itself. CSV built-in functions ignore this option."
+ },
+ "encoding": {
+ "type": "string",
+ "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. For reading, decodes the CSV files by the given encoding type. CSV built-in functions ignore this option."
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json
new file mode 100644
index 0000000..af783f6
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json
@@ -0,0 +1,17 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/spark_json_reader.schema.json",
+ "title": "Keyword Arguments for SparkJSONReader",
+ "description": "Arguments to contol how the Spark JSON Reader interacts with a json file",
+ "type": "object",
+ "properties": {
+ "encoding": {
+ "type": "string",
+ "description": "For reading, allows to forcibly set one of standard basic or extended encoding for the JSON files. For example UTF-16BE, UTF-32LE."
+ },
+ "multi_line": {
+ "type": "boolean",
+ "description": "Parse one record, which may span multiple lines, per file. JSON built-in functions ignore this option."
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json
new file mode 100644
index 0000000..9ab1c74
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json
@@ -0,0 +1,43 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/spark_xml_reader.schema.json",
+ "title": "Keyword Arguments for SparkXMLReader",
+ "description": "Arguments to contol how the Spark XML Reader interacts with a xml file",
+ "type": "object",
+ "anyOf": [
+ {
+ "$ref": "global_xml_reader_args.schema.json"
+ }
+ ],
+ "properties": {
+ "sampling_ratio": {
+ "type": "integer",
+ "description": "Sampling ratio for inferring schema (0.0 ~ 1)."
+ },
+ "exclude_attribute": {
+ "type": "boolean",
+ "description": "Whether you want to exclude attributes in elements or not."
+ },
+ "mode": {
+ "type": "string",
+ "description": "The mode for dealing with corrupt records during parsing. Permissive = When it encounters a corrupted record, it sets all fields to null and puts the malformed string into a new field configured by columnNameOfCorruptRecord. When it encounters a field of the wrong datatype, it sets the offending field to null; DROPMALFORMED = ignores the whole corrupted records; FAILFAST = throws an exception when it meets corrupted records.",
+ "enum": [
+ "PERMISSIVE",
+ "FAILFAST",
+ "DROPMALFORMED"
+ ]
+ },
+ "infer_schema": {
+ "type": "boolean",
+ "description": "If true, attempts to infer an appropriate type for each resulting DataFrame column, like a boolean, numeric or date type. If false, all resulting columns are of string type."
+ },
+ "ignore_namespace": {
+ "type": "boolean",
+ "description": "If true, namespaces prefixes on XML elements and attributes are ignored. Tags and would, for example, be treated as if both are just . Note that, at the moment, namespaces cannot be ignored on the rowTag element, only its children. Note that XML parsing is in general not namespace-aware even if false."
+ },
+ "namespace": {
+ "type": "string",
+ "description": "Applies a namespace to each rowTag if required."
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json
new file mode 100644
index 0000000..da7cdbc
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json
@@ -0,0 +1,12 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/reader_constraints/spark_xml_stream_reader.schema.json",
+ "title": "Keyword Arguments for SparkXMLStreamReader",
+ "description": "Arguments to contol how the Spark XML Stream Reader interacts with a xml file",
+ "type": "object",
+ "anyOf": [
+ {
+ "$ref": "global_xml_reader_args.schema.json"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json
new file mode 100644
index 0000000..4509b3d
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json
@@ -0,0 +1,17 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/identifier.schema.json",
+ "title": "Constraints for alphanumeric value",
+ "description": "Constaints for an Alphanumeric type",
+ "type": "object",
+ "properties": {
+ "min_digits": {
+ "type": "integer",
+ "description": "The minimum number of alphanumeric characters required"
+ },
+ "max_digits": {
+ "type": "integer",
+ "description": "The maximum number of alphanumeric characters required"
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json
new file mode 100644
index 0000000..a25ba1c
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json
@@ -0,0 +1,25 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/condate.schema.json",
+ "title": "Constraint for Dates",
+ "description": "Constaints for an Date type",
+ "type": "object",
+ "properties": {
+ "gt": {
+ "type": "string",
+ "description": "Enforces date to be greater than the set value"
+ },
+ "ge": {
+ "type": "string",
+ "description": "Enforces date to be greater than or equal to the set value"
+ },
+ "lt": {
+ "type": "string",
+ "description": "Enforces date to be less than the set value"
+ },
+ "le": {
+ "type": "string",
+ "description": "Enforces date to be less than or equal to the set value"
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json
new file mode 100644
index 0000000..4ea552c
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json
@@ -0,0 +1,37 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/condecimal.schema.json",
+ "title": "Constraints for Decimals",
+ "description": "Constaints for an Decimal type",
+ "type": "object",
+ "properties": {
+ "gt": {
+ "type": "number",
+ "description": "Enforces decimal to be greater than the set value"
+ },
+ "ge": {
+ "type": "number",
+ "description": "Enforces decimal to be greater than or equal to the set value"
+ },
+ "lt": {
+ "type": "number",
+ "description": "Enforces decimal to be less than the set value"
+ },
+ "le": {
+ "type": "number",
+ "description": "Enforces decimal to be less than or equal to the set value"
+ },
+ "max_digits": {
+ "type": "integer",
+ "description": "Maximum number of digits within the decimal. it does not include a zero before the decimal point or trailing decimal zeroes"
+ },
+ "decimal_places": {
+ "type": "integer",
+ "description": "Max number of decimal places allowed. it does not include trailing decimal zeroes"
+ },
+ "multiple_of": {
+ "type": "number",
+ "description": "Enforces decimal to be a multiple of the set value"
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json
new file mode 100644
index 0000000..40fe854
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json
@@ -0,0 +1,34 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/conformatteddate.schema.json",
+ "title": "Constraints for Constrained Formatted Date",
+ "description": "Constaints for an conformatteddate type",
+ "type": "object",
+ "properties": {
+ "date_format": {
+ "type": "string",
+ "description": "Enforces date to match a specific format."
+ },
+ "strict": {
+ "type": "boolean",
+ "description": "Controls type coercion"
+ },
+ "ge": {
+ "type": "string",
+ "description": "Enforces date to be greater than or equal to the set value"
+ },
+ "le": {
+ "type": "string",
+ "description": "Enforces date to be less than the set value"
+ },
+ "gt": {
+ "type": "string",
+ "description": "Enforces date to be greater than the set value"
+ },
+ "lt": {
+ "type": "string",
+ "description": "Enforces date to be less than or equal to the set value"
+ }
+ },
+ "required": ["date_format"]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json
new file mode 100644
index 0000000..7de2c53
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json
@@ -0,0 +1,33 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/conint.schema.json",
+ "title": "Constraints for Integer",
+ "description": "Constaints for an Integer type",
+ "type": "object",
+ "properties": {
+ "strict": {
+ "type": "boolean",
+ "description": "Controls type coercion"
+ },
+ "gt": {
+ "type": "integer",
+ "description": "Enforces integer to be greater than the set value"
+ },
+ "ge": {
+ "type": "integer",
+ "description": "Enforces integer to be greater than or equal to the set value"
+ },
+ "lt": {
+ "type": "integer",
+ "description": "Enforces integer to be less than the set value"
+ },
+ "le": {
+ "type": "integer",
+ "description": "Enforces integer to be less than or equal to the set value"
+ },
+ "multiple_of": {
+ "type": "integer",
+ "description": "Enforces integer to be a multiple of the set value"
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json
new file mode 100644
index 0000000..d9fd393
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json
@@ -0,0 +1,33 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/constr.schema.json",
+ "title": "Constraints for String",
+ "description": "Constaints for an String type",
+ "type": "object",
+ "properties": {
+ "strip_whitespace": {
+ "type": "boolean",
+ "description": "Removes leading and trailing whitespace"
+ },
+ "to_lower": {
+ "type": "boolean",
+ "description": "Turns all characters to lowercase"
+ },
+ "min_length": {
+ "type": "integer",
+ "description": "Minimum length of the byte string"
+ },
+ "max_length": {
+ "type": "integer",
+ "description": "Maximum length of the byte string"
+ },
+ "strict": {
+ "type": "boolean",
+ "description": "Controls type coercion"
+ },
+ "regex": {
+ "type": "string",
+ "description": "Enforces string to match a specific regex pattern"
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json
new file mode 100644
index 0000000..35691de
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json
@@ -0,0 +1,21 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/formatteddatetime.schema.json",
+ "title": "Constraints for Formatted Datetime",
+ "description": "Constaints for an FormattedDatetime type",
+ "type": "object",
+ "properties": {
+ "date_format": {
+ "type": "string",
+ "description": "Enforces date to match a specific format."
+ },
+ "timezone_treatment": {
+ "type": "string",
+ "enum": [
+ "forbid",
+ "permit",
+ "require"
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json
new file mode 100644
index 0000000..c827730
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json
@@ -0,0 +1,21 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/formattedtime.schema.json",
+ "title": "Constraints for Formatted Time",
+ "description": "Constaints for an FormattedTime type",
+ "type": "object",
+ "properties": {
+ "time_format": {
+ "type": "string",
+ "description": "Enforces date to match a specific format."
+ },
+ "timezone_treatment": {
+ "type": "string",
+ "enum": [
+ "forbid",
+ "permit",
+ "require"
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json
new file mode 100644
index 0000000..d8323da
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json
@@ -0,0 +1,12 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/identifier.schema.json",
+ "title": "Constraints for alphanumeric identifier (including fixed set of special characters)",
+ "description": "Constaints for an Identifer type (alphanumeric + fixed set of special chars)",
+ "type": "object",
+ "allOf": [
+ {
+ "$ref": "alphanumeric.schema.json"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json
new file mode 100644
index 0000000..50272ca
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json
@@ -0,0 +1,7 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/nhsnumber.schema.json",
+ "title": "Constraints for NHSNumber",
+ "description": "Constaints for the NHSNumber type",
+ "type": "object"
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json
new file mode 100644
index 0000000..a7b9a33
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json
@@ -0,0 +1,17 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/orgid.schema.json",
+ "title": "Constraints for Organisation Idenitifer such as ODS Code",
+ "description": "Constaints for an OrgId type",
+ "type": "object",
+ "properties": {
+ "pattern": {
+ "type": "string",
+ "description": "Enforces string to match a specific regex pattern"
+ },
+ "strip_whitespace": {
+ "type": "boolean",
+ "description": "Removes leading and trailing whitespace"
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json
new file mode 100644
index 0000000..4e0eeba
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json
@@ -0,0 +1,19 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/permissivenhsno.schema.json",
+ "title": "Constraints for NHSNumber",
+ "description": "Constaints for the NHSNumber type",
+ "type": "object",
+ "allOf": [
+ {
+ "$ref": "constr.schema.json"
+ }
+ ],
+ "properties": {
+ "warn_on_test_numbers" : {
+ "type": "boolean",
+ "description": "Whether only warning should be raised if test numbers included"
+ }
+ },
+ "additionalProperties": false
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json
new file mode 100644
index 0000000..31685fc
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json
@@ -0,0 +1,22 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/postcode.schema.json",
+ "title": "Constraints for Postcode",
+ "description": "Constaints for an Postcode type. Defaults to UK postcode.",
+ "type": "object",
+ "allOf": [
+ {
+ "$ref": "constr.schema.json"
+ }
+ ],
+ "properties": {
+ "pattern": {
+ "type": "string",
+ "description": "Regex format for the postcode"
+ },
+ "apply_normalize": {
+ "type": "boolean",
+ "description": "Whether to remove whitespace from the input value"
+ }
+ }
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json
new file mode 100644
index 0000000..d2a7f0e
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json
@@ -0,0 +1,22 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/reportingperiodend.schema.json",
+ "title": "Constraints for Reporting Period End Date",
+ "description": "Constaints for an ReportingPeriod (end) type",
+ "type": "object",
+ "properties": {
+ "reporting_period_type": {
+ "type": "string",
+ "description": "Enforces date to be the start or end of a month.",
+ "enum": [
+ "start",
+ "end"
+ ]
+ },
+ "date_format": {
+ "type": "string",
+ "description": "Enforces date to match a specific format"
+ }
+ },
+ "required": ["reporting_period_type"]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json
new file mode 100644
index 0000000..f5749c2
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json
@@ -0,0 +1,22 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:contract/components/types/reportingperiodstart.schema.json",
+ "title": "Constraints for Reporting Period Start Date",
+ "description": "Constaints for an ReportingPeriod (start) type",
+ "type": "object",
+ "properties": {
+ "reporting_period_type": {
+ "type": "string",
+ "description": "Enforces date to be the start or end of a month.",
+ "enum": [
+ "start",
+ "end"
+ ]
+ },
+ "date_format": {
+ "type": "string",
+ "description": "Enforces date to match a specific format"
+ }
+ },
+ "required": ["reporting_period_type"]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json
index 0f331ef..660a910 100644
--- a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json
@@ -1,7 +1,29 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/components/type_name.schema.json",
"title": "type_name",
"description": "The name of an allowed type within data ingest. This should be a Python type",
- "type": "string"
-}
+ "type": "string",
+ "enum": [
+ "str",
+ "int",
+ "float",
+ "date",
+ "datetime",
+ "constr",
+ "conint",
+ "condate",
+ "condecimal",
+ "postcode",
+ "nhsnumber",
+ "permissivenhsno",
+ "alphanumeric",
+ "identifier",
+ "orgid",
+ "formatteddatetime",
+ "formattedtime",
+ "conformatteddate",
+ "reportingperiodstart",
+ "reportingperiodend"
+ ]
+}
\ No newline at end of file
diff --git a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json b/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json
deleted file mode 100644
index 69f0f85..0000000
--- a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "$schema": "https://json-schema.org/draft-07/schema",
- "$id": "data-ingest:contract/components/validation_function.schema.json",
- "title": "validation_function",
- "description": "A validation function to be applied to a field",
- "type": "object",
- "properties": {
- "name": {
- "description": "The name of the validation function to be applied",
- "type": "string"
- },
- "error_type": {
- "description": "The type of error to emit if the function fails. This defaults to 'record_rejection'",
- "type": "string",
- "enum": [
- "record_rejection",
- "file_rejection",
- "warning"
- ]
- },
- "error_message": {
- "description": "The message to associate with the error, if the function fails",
- "type": "string"
- },
- "kwargs": {
- "description": "Keyword arguments to be passed to the Python function",
- "type": "object"
- }
- },
- "required": [
- "name"
- ],
- "additionalProperties": false
-}
diff --git a/docs/advanced_guidance/json_schemas/contract/contract.schema.json b/docs/advanced_guidance/json_schemas/contract/contract.schema.json
index dad5a92..f80a6e5 100644
--- a/docs/advanced_guidance/json_schemas/contract/contract.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/contract.schema.json
@@ -1,8 +1,8 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:contract/contract.schema.json",
"title": "contract",
- "description": "A data contract within data ingest. This specifies the information required to read in the data",
+ "description": "A data contract within data ingest. This specifies the information required to model and read the data",
"type": "object",
"properties": {
"types": {
@@ -24,12 +24,15 @@
"type": "object",
"additionalProperties": {
"$ref": "components/readable_entity.schema.json"
- }
+ },
+ "minProperties": 1
},
"error_details": {
- "description": "Relative path from the dischema document to reference a error details document.",
- "type": "string"
+ "description": "Relative path from the dischema document to reference an error details document.",
+ "type": "string",
+ "pattern": "^[a-zA-Z0-9].*_data_contract_error_details\\.json$"
}
},
+ "required": ["datasets"],
"additionalProperties": true
}
diff --git a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json
index 58c369b..046d751 100644
--- a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json
+++ b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json
@@ -22,6 +22,7 @@
"$defs": {
"errorObject": {
"type": "object",
+ "required": ["error_code", "error_message"],
"properties": {
"error_code": {
"type": "string",
diff --git a/docs/advanced_guidance/json_schemas/dataset.schema.json b/docs/advanced_guidance/json_schemas/dataset.schema.json
index 75cf4c3..4e85011 100644
--- a/docs/advanced_guidance/json_schemas/dataset.schema.json
+++ b/docs/advanced_guidance/json_schemas/dataset.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:dataset.schema.json",
"title": "dataset",
"description": "A full dataset for data ingest. This contains the contract and the associated transformations",
diff --git a/docs/advanced_guidance/json_schemas/rule_store.schema.json b/docs/advanced_guidance/json_schemas/rule_store.schema.json
index cf27308..a3822bb 100644
--- a/docs/advanced_guidance/json_schemas/rule_store.schema.json
+++ b/docs/advanced_guidance/json_schemas/rule_store.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:rule_store.schema.json",
"title": "rule_store",
"description": "A store of rules within data ingest. Property names should match the names of the contained rules.",
diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json
index d962165..372caa5 100644
--- a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/components/business_filter.schema.json",
"title": "rule",
"description": "Descriptions of a 'business' filter within data ingest",
@@ -10,28 +10,6 @@
},
"parameters": {
"type": "object"
- },
- "error_code": {
- "type": "string"
- },
- "error_location": {
- "type": "string"
- },
- "reporting_entity": {
- "type": "string"
- },
- "reporting_field": {
- "type": [
- "array",
- "string"
- ],
- "items": {
- "type": "string"
- }
- },
- "reporting_field_name": {
- "description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.",
- "type": "string"
}
},
"required": [
diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json
index 6aae36d..07449d0 100644
--- a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/components/business_rule.schema.json",
"title": "rule",
"description": "Definition of a business rule within data ingest",
@@ -9,10 +9,12 @@
"description": "A templateable business rule.",
"properties": {
"rule_name": {
- "type": "string"
+ "type": "string",
+ "description": "Name of the rule"
},
"description": {
- "type": "string"
+ "type": "string",
+ "description": "Description of the rule"
},
"type": {
"const": "transformation"
@@ -21,10 +23,12 @@
"type": "object",
"additionalProperties": {
"type": "string"
- }
+ },
+ "description": "Descriptions of the parameters"
},
"parameter_defaults": {
- "type": "object"
+ "type": "object",
+ "description": "Defaults for the parameters if not explicitly supplied"
},
"rule_config": {
"$ref": "rule.schema.json"
@@ -39,10 +43,12 @@
"description": "A templateable business filter.",
"properties": {
"rule_name": {
- "type": "string"
+ "type": "string",
+ "description": "Name of the rule"
},
"description": {
- "type": "string"
+ "type": "string",
+ "description": "Description of the rule"
},
"type": {
"const": "filter"
@@ -51,10 +57,12 @@
"type": "object",
"additionalProperties": {
"type": "string"
- }
+ },
+ "description": "Descriptions of the parameters"
},
"parameter_defaults": {
- "type": "object"
+ "type": "object",
+ "description": "Defaults for the parameters if not explicitly supplied"
},
"rule_config": {
"$ref": "filter.schema.json"
@@ -69,10 +77,12 @@
"description": "A complex rule.",
"properties": {
"rule_name": {
- "type": "string"
+ "type": "string",
+ "description": "Name of the rule"
},
"description": {
- "type": "string"
+ "type": "string",
+ "description": "Description of the rule"
},
"type": {
"const": "complex_rule"
@@ -81,10 +91,12 @@
"type": "object",
"additionalProperties": {
"type": "string"
- }
+ },
+ "description": "Descriptions of the parameters"
},
"parameter_defaults": {
- "type": "object"
+ "type": "object",
+ "description": "Defaults for the parameters if not explicitly supplied"
},
"rule_config": {
"type": "object",
diff --git a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json
index a1dfd2e..c27ff08 100644
--- a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/components/concrete_filter.schema.json",
"title": "rule",
"description": "Description of a filter within data ingest. This filter uses a SQL expression, and requires a name.",
diff --git a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json
index 409ee98..e24da53 100644
--- a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json
@@ -1,18 +1,21 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/components/core_filter.schema.json",
"title": "rule",
"description": "Descriptions of a filter within data ingest, implemented using a SQL expression. This filter does not require a name, but a concrete filter does.",
"type": "object",
"properties": {
"entity": {
- "type": "string"
+ "type": "string",
+ "description": "Name of the entity to perform the rule against"
},
"name": {
- "type": "string"
+ "type": "string",
+ "description": "Name for the filter"
},
"expression": {
- "type": "string"
+ "type": "string",
+ "description": "SQL expression for the filter"
},
"failure_type": {
"enum": [
@@ -20,10 +23,12 @@
"submission",
"integrity",
"group"
- ]
+ ],
+ "description": "Type of failure"
},
"failure_message": {
- "type": "string"
+ "type": "string",
+ "description": "Customise the error message generated"
},
"is_informational": {
"anyOf": [
@@ -38,7 +43,8 @@
"{{ is_informational }}"
]
}
- ]
+ ],
+ "description": "Whether the filter is a warning or rejection"
},
"error_code": {
"anyOf": [
@@ -48,13 +54,16 @@
{
"type": "number"
}
- ]
+ ],
+ "description": "Customise the error code generated"
},
"error_location": {
- "type": "string"
+ "type": "string",
+ "description": "Utilised where reporting_field not defined. Should represent the field/column that triggered the rule"
},
"reporting_entity": {
- "type": "string"
+ "type": "string",
+ "description": "Allows customisation over the name of the entity that triggered the validation rule"
},
"reporting_field": {
"type": [
@@ -63,17 +72,26 @@
],
"items": {
"type": "string"
- }
+ },
+ "description": "The field(s) that triggered the validation rule"
},
"reporting_field_name": {
"description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.",
"type": "string"
},
"category": {
- "type": "string"
+ "type": "string",
+ "enum": [
+ "Blank",
+ "Wrong format",
+ "Bad value",
+ "Bad file"
+ ],
+ "description": "Category of failure"
},
- "other_entities": {
- "type": "string"
+ "original_entity_override": {
+ "type": "string",
+ "description": "Allows the overriding of the entity to source the id/key field(s) from"
}
},
"required": [
diff --git a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json
index 54809ab..598dd23 100644
--- a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/components/filter.schema.json",
"title": "rule",
"description": "Description of a 'real' filter within data ingest",
diff --git a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json
index 44cba18..475117b 100644
--- a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/components/multiple_expressions.schema.json",
"title": "multiple_expressions",
"description": "A definition of multiple aliased expressions.",
diff --git a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json
index 2e54fad..18af5a7 100644
--- a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/components/rule.schema.json",
"title": "rule",
"description": "Descriptions of allowed rules within data ingest",
diff --git a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json
index e68e175..2c64538 100644
--- a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json
+++ b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json
@@ -1,5 +1,5 @@
{
- "$schema": "https://json-schema.org/draft-07/schema",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "data-ingest:transformations/transformations.schema.json",
"title": "transformations",
"description": "Transformations within data ingest. This specifies the information required to transform and filter the data",
@@ -16,31 +16,47 @@
"oneOf": [
{
"type": "object",
+ "additionalProperties": false,
"properties": {
+ "type": {
+ "const": "filename"
+ },
"filename": {
"type": "string",
- "description": "The path to the reference data (as Parquet) relative to the contract"
+ "description": "The path to the reference data (as Parquet) relative to the contract",
+ "pattern": ".*\\.(?:arrow|parquet)$"
}
},
"required": [
+ "type",
"filename"
]
},
{
"type": "object",
+ "additionalProperties": false,
"properties": {
+ "type": {
+ "const": "uri"
+ },
"uri": {
"type": "string",
- "description": "The absolute URI of the reference data (as Parquet)"
+ "description": "The absolute URI of the reference data (as Parquet or Arrow)",
+ "pattern": "^[a-zA-Z][a-zA-Z0-9+.-]*:.+\\.(?:arrow|parquet)$"
}
},
"required": [
+ "type",
"uri"
]
},
{
"type": "object",
+ "additionalProperties": false,
"properties": {
+ "type": {
+ "const": "database"
+ },
"database": {
"type": "string",
"description": "The database the reference data is stored in.",
@@ -49,10 +65,12 @@
"table_name": {
"type": "string",
"description": "The table name (excluding database) containing the reference data.",
- "pattern": "^(?:(?refdata)_)?(?[A-Za-z0-9]+)_(?\\w+)$"
+ "pattern": "^\\w+$"
}
},
"required": [
+ "type",
+ "database",
"table_name"
]
}
@@ -63,7 +81,8 @@
"description": "Stores of business rules",
"type": "array",
"items": {
- "anyOf": [
+ "type": "object",
+ "oneOf": [
{
"properties": {
"store_type": {
@@ -72,17 +91,29 @@
"rules": {
"$ref": "../rule_store.schema.json"
}
- }
+ },
+ "required": [
+ "store_type",
+ "rules"
+ ],
+ "additionalProperties": false
},
{
"properties": {
"store_type": {
- "type": "string"
+ "const": "json"
},
"filename": {
- "type": "string"
+ "type": "string",
+ "description": "Relative path to the current dischema document containing the rulestore",
+ "pattern": "^.*\\.json$"
}
- }
+ },
+ "required": [
+ "store_type",
+ "filename"
+ ],
+ "additionalProperties": false
}
]
}
@@ -94,7 +125,8 @@
"type": "object",
"properties": {
"rule_name": {
- "type": "string"
+ "type": "string",
+ "minLength": 1
},
"parameters": {
"type": "object"
@@ -143,4 +175,4 @@
},
"additionalProperties": false
}
-}
+}
\ No newline at end of file