From 9f88d5becd5740dac13934351185d416a3d28c89 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:04:11 +0100 Subject: [PATCH 01/11] docs: clarify supported os --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8ae030..8e08264 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Operating System :: OS Independent", + "Operating System :: POSIX :: Linux", "Topic :: Software Development :: Libraries", "Typing :: Typed", ] From 79e5af1b5fc81cd1ed50c6c58b981398e00964db Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:31:09 +0100 Subject: [PATCH 02/11] docs: update readme for v0.9 --- README.md | 49 +++++++++++++------------------------------------ 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 7c16bb5..7f1b5fc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- + Data Validation Engine

@@ -9,49 +9,26 @@ [![CI Unit Tests](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml) [![CI Formatting & Linting](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml) -The Data Validation Engine (DVE) is a configuration driven data validation library built and utilised by NHS England. Currently the package has been reverted from v1.0.0 release to a 0.x as we feel the package is not yet mature enough to be considered a 1.0.0 release. So please bear this in mind if reading through the commits and references to a v1+ release when on v0.x. +The Data Validation Engine (DVE) is a configuration-driven data validation library created and used by NHS England. It lets users define validation rules once and apply them across multiple dataset collections - supporting consistent, accurate data checks. -As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` file and example files can be accessed [here](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here](https://nhsdigital.github.io/data-validation-engine/). +__The DVE offers__: -Once a dischema file has been defined, you are ready to use the DVE. The DVE is typically orchestrated based on four key "services". These are... +- SQL configuration-based validations +- Format normalization to Parquet for a unified data representation +- Data modelling and typecasting +- Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends +- Clear validation reporting, including summary insights and record-level error messages -| | Service | Purpose | -| -- | ------- | ------- | -| 1. | File Transformation | This service will take submitted files and turn them into stringified parquet file(s) to ensure that a consistent data structure can be passed through the other services. | -| 2. | Data Contract | This service will validate and perform type casting against a stringified parquet file using [pydantic models](https://docs.pydantic.dev/1.10/). | -| 3. | Business Rules | The business rules service will perform more complex validations such as comparisons between fields and tables, aggregations, filters etc to generate new entities. | -| 4. | Error Reports | The error reports service will take all the errors raised in previous services and surface them into a readable format for a downstream users/service. Currently, this implemented to be an excel spreadsheet but could be reconfigured to meet other requirements/use cases. | - -If you'd like more detailed documentation around these services the please read the extended documentation [here](https://nhsdigital.github.io/data-validation-engine/). - -The DVE has been designed in a way that's modular and can support users who just want to utilise specific "services" from the DVE (i.e. just the file transformation + data contract). Additionally, the DVE is designed to support different backend implementations. As part of the base installation of DVE, you will find backend support for `Spark` and `DuckDB`. So, if you need a `MySQL` backend implementation, you can implement this yourself. Given our organisations requirements, it will be unlikely that we add anymore specific backend implementations into the base package beyond Spark and DuckDB. So, if you are unable to implement this yourself, I would recommend reading the guidance on [requesting new features and raising bug reports here](#requesting-new-features-and-raising-bug-reports). - -Additionally, if you'd like to contribute a new backend implementation into the base DVE package, then please look at the [Contributing](#Contributing) section. +As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` (data ingest schema) file and example files can be accessed [here ↗️](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/). ## Installation and usage -The DVE is a Python package and can be installed using package managers such as [pip](https://pypi.org/project/pip/). As of the latest release we support Python 3.10 & 3.11, with Spark v3.4 and DuckDB v1.1. In the future we will be looking to upgrade the DVE to working on a higher versions of Python, DuckDB and Spark. - -If you're planning to use the Spark backend implementation, you will also need OpenJDK 11 installed. - -Python dependencies are listed in `pyproject.toml`. - -To install the DVE package you can simply install using a package manager such as [pip](https://pypi.org/project/pip/). - -``` -pip install data-validation-engine -``` - -*Note - Only versions >=0.6.2 are available on PyPi. For older versions please install directly from the git repo or build from source.* - -Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema JSON document (configuration), please read the [documentation](https://nhsdigital.github.io/data-validation-engine/). - -Version 0.0.1 does support a working Python 3.7 installation. However, we will not be supporting any issues with that version of the DVE if you choose to use it. __Use at your own risk__. +Please see the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/user_guidance/install/). ## Requesting new features and raising bug reports **Before creating new issues, please check to see if the same bug/feature has been created already. Where a duplicate is created, the ticket will be closed and referenced to an existing issue.** -If you have spotted a bug with the DVE then please raise an issue [here](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template". +If you have spotted a bug with the DVE then please raise an issue [here ↗️](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template". If you have feature request then please follow the same process whilst using the "Feature request template". @@ -63,7 +40,7 @@ Below is a list of features that we would like to implement or have been request | Uplift to Python 3.11 | 0.2.0 | Yes | | Uplift Pyspark to 3.5 | 0.8.0 | Yes | | Allow DVE to run on Python 3.12+ | 0.8.0 | Yes | -| Upgrade to Pydantic 2.0 | 0.9.0 | No | +| Upgrade to Pydantic 2.0 | 0.9.0 | Yes | | Uplift Pyspark to 4.0+ | TBA | No | | Polars upgrade to v1+ | TBA | No | | DuckDB upgrade to v1.5+ | TBA | No | @@ -73,7 +50,7 @@ Below is a list of features that we would like to implement or have been request If you are interested in getting any of the unreleased features listed above available, then please read the [Contributing](#Contributing) section and then submit us a pull request. ## Contributing -Please see guidance [here](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md). +Please see guidance [here ↗️](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md). ## Legal This codebase is released under the MIT License. This covers both the codebase and any sample code in the documentation. From 88a076aca6d153f6b677f72f25c9128369ce6909 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:38:30 +0100 Subject: [PATCH 03/11] docs: add security.md --- SECURITY.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..abb9914 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,23 @@ +# Security + +We take security and the protection of private data extremely seriously. If you believe you have found a vulnerability or other issue which has compromised or could compromise the security of any of our systems or private data managed by our systems, please do not hesitate to contact us using the method outlined below. + +## Table of contents + +- [Security](#security) +- [Table of contents](#table-of-contents) +- [Reporting a vulnerability](#reporting-a-vulnerability) +- [General security enquiries](#general-security-enquiries) + +## Reporting a vulnerability + +If you believe you have found a security issue in this repository, please report it using GitHub's private vulnerability reporting: + +1. [Report a vulnerability](https://github.com/NHSDigital/data-validation-engine/security/advisories/new) +2. Provide details of the issue and steps to reproduce + +This creates a private channel for discussion and allows us to coordinate a fix before any public disclosure. + +## General security enquiries + +If you have general enquiries regarding our cybersecurity, please reach out to us at [cybersecurity@nhs.net](cybersecurity@nhs.net) From ceacb3e446c5c0e52d764091bc46aeef1f512909 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:40:09 +0100 Subject: [PATCH 04/11] docs: rename CONTRIBUTE.md --- CONTRIBUTE.md => CONTRIBUTING.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CONTRIBUTE.md => CONTRIBUTING.md (100%) diff --git a/CONTRIBUTE.md b/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTE.md rename to CONTRIBUTING.md From 44b635093864a3aeda0a82f8f2d90a9932a1d6b7 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:19:15 +0100 Subject: [PATCH 05/11] build: update docs around references to pydantic v1 --- docs/user_guidance/data_contract.md | 18 +++++++----------- docs/user_guidance/install.md | 25 ++++++++++--------------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/docs/user_guidance/data_contract.md b/docs/user_guidance/data_contract.md index 55913da..13e72ef 100644 --- a/docs/user_guidance/data_contract.md +++ b/docs/user_guidance/data_contract.md @@ -6,11 +6,7 @@ tags: - Domain Types --- -The Data Contract defines the structure (models) of your data and controls how it is typecast. We use [Pydantic](https://pydantic.dev/docs/validation/1.10/overview/) to generate and validate the models. This page is meant to give you greater details on how you should write your Data Contract. If you want a summary of how the Data Contract works, please refer to the [Getting Started](./getting_started.md#rules-configuration-introduction) page. - -!!! Note - - We plan to migrate to Pydantic v2+ in v0.9.0. This page currently reflects what is available through Pydantic v1. +The Data Contract defines the structure (models) of your data and controls how it is typecast. We use [Pydantic v2](https://pydantic.dev/docs/validation/latest/get-started/) to generate and validate the models. This page is meant to give you greater details on how you should write your Data Contract. If you want a summary of how the Data Contract works, please refer to the [Getting Started](./getting_started.md#rules-configuration-introduction) page. ## Models @@ -207,15 +203,15 @@ If you want to read more about the readers, please see the [File Transformation] Within the `fields` section of the contract you must define what data type a given field should be. Depending on how strict/lenient you want your types to be, a number of types are available to use. The types available are: - [Built-in standard library](https://docs.python.org/3.12/library/stdtypes.html) types (such as `int`, `str`, `date`) available with your version of Python installed for the DVE. -- [Pydantic v1 types](https://docs.pydantic.dev/1.10/usage/types/) +- [Pydantic v2 types](https://pydantic.dev/docs/validation/latest/concepts/types/) - [Custom Types](./data_contract.md#custom-types) - [Domain types](./data_contract.md#domain-types) ### Constraints -Given the DVE supports Pydantic types, you can use any of the [constrained types available](https://docs.pydantic.dev/1.10/usage/types/#constrained-types). The Pydantic docs will also show you what `kwarg` arguments are available for each constraint such as min/max length, regex patterns etc. +The DVE supports constraints against valid python/pydantic types. You can use the `constraints` arguments to apply a specifc constraint such as min/max length etc. -For example, if you wanted to use a `constr` type for a field, you would define it like this: +For example, if you wanted to use apply constraints to a `str` type, you may define it like this: ```json title="movies.dischema.json" { @@ -225,7 +221,7 @@ For example, if you wanted to use a `constr` type for a field, you would define "fields": { "movie_uuid": "int", "movie_name": { - "callable": "constr", + "callable": "str", "constraints": { "min_length": 1, "max_length": 20 @@ -249,7 +245,7 @@ As shown in the [Constraints](./data_contract.md#constraints) section above, you { "types": { "MyConstrainedString": { - "callable": "constr", + "callable": "str", "constraints": { "min_length": 1, "max_length": 20 @@ -334,7 +330,7 @@ To define a struct type, you would add it to the `types` section like this... "fields": { "movie_uuid": "int", "movie_name": { - "callable": "constr", + "callable": "str", "constraints": { "min_length": 1, "max_length": 20 diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md index e0c2ce4..893b229 100644 --- a/docs/user_guidance/install.md +++ b/docs/user_guidance/install.md @@ -8,11 +8,11 @@ tags: !!! warning **DVE is currently an unstable package. Expect breaking changes between every minor patch**. We intend to follow semantic versioning of `major.minor.patch` more strictly after a 1.0 release. Until then, we recommend that you pin your install to the latest version available and keep an eye on [future releases](https://github.com/NHSDigital/data-validation-engine/releases). - **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.12, but it's not an immediate priority. + **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.13, but it's not an immediate priority. If working on Python 3.7, the `0.1` release supports this (and only this) version of Python. However, we have not been updating that version with any bugfixes, performance improvements etc. There are also a number of vulnerable dependencies on version `0.1` release due to [Python 3.7 being depreciated](https://devguide.python.org/versions/) and a number of packages dropping support. **If you choose to install `0.1`, you accept the risks of doing so and additional support will not be provided.** -You can install the DVE package through python package managers such as [pip](https://pypi.org/project/pip/), [pipx](https://github.com/pypa/pipx), [uv](https://docs.astral.sh/uv/) and [poetry](https://python-poetry.org/). +You can install the DVE package through python package managers such as [pip](https://pypi.org/project/pip/), [uv](https://docs.astral.sh/uv/) and [poetry](https://python-poetry.org/). === "pip" @@ -20,12 +20,6 @@ You can install the DVE package through python package managers such as [pip](ht pip install data-validation-engine ``` -=== "pipx" - - ```sh - pipx install data-validation-engine - ``` - === "uv" Add to your existing `uv` project... @@ -84,10 +78,11 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru ## DVE Version Compatability Matrix -| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | -| ------------ | -------------- | -------------- | ------------- | ---------------- | -| >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 | -| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 | -| >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 | -| >=0.2,<0.6 | >=3.10,<3.12 | 1.1.0 | 3.4.4 | 1.10.15 | -| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 | +| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | +| ------------ | -------------- | -------------- | --------------- | ---------------- | +| >=0.9.0 | >=3.10,<3.12 | 1.1.3 | >=3.5.0,<=3.5.5 | 2.13.4 | +| >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 | +| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 | +| >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 | +| >=0.2,<0.6 | >=3.10,<3.12 | 1.1.0 | 3.4.4 | 1.10.15 | +| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 | From 76f88d6f253896221d94caa89fc751a8aed66494 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:58:16 +0100 Subject: [PATCH 06/11] docs: clarify python version --- docs/user_guidance/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md index 893b229..5ad4d50 100644 --- a/docs/user_guidance/install.md +++ b/docs/user_guidance/install.md @@ -8,7 +8,7 @@ tags: !!! warning **DVE is currently an unstable package. Expect breaking changes between every minor patch**. We intend to follow semantic versioning of `major.minor.patch` more strictly after a 1.0 release. Until then, we recommend that you pin your install to the latest version available and keep an eye on [future releases](https://github.com/NHSDigital/data-validation-engine/releases). - **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.13, but it's not an immediate priority. + **Please note that we only support Python runtimes of 3.10, 3.11 & 3.12.** In the future we will look to add support for Python versions greater than 3.12, but it's not an immediate priority. If working on Python 3.7, the `0.1` release supports this (and only this) version of Python. However, we have not been updating that version with any bugfixes, performance improvements etc. There are also a number of vulnerable dependencies on version `0.1` release due to [Python 3.7 being depreciated](https://devguide.python.org/versions/) and a number of packages dropping support. **If you choose to install `0.1`, you accept the risks of doing so and additional support will not be provided.** From 5aa2dc8508193b9382d55519ac27f31a7a9fc490 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:38:54 +0100 Subject: [PATCH 07/11] docs: add faq page --- docs/faq.md | 24 ++++++++++++++++++++++++ zensical.toml | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 docs/faq.md diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..af25fbe --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,24 @@ +--- +title: Frequently Asked Questions (FAQ) +tags: + - FAQ + - Business Rules + - Error Reports +--- + +??? question "The `id` field in the error report is missing when it should be present?" + + If the `id` field isn't populated then it will be due to the fact that the key field is not present in the entity you are looking up against. The `key_fields`/`key_fields` is sourced from a dictionary lookup sourced in the data contract, so new entities defined within the business_rules won't have this lookup and it will result in the column not being populated. To get around this issue for entities without a defined `key_field` you can use the `original_entity_override` key to define the entity you want to source your `key_field`/`key_fields` from. You may find an example below. + + ??? example + + ```json + "filters": [ + { + "entity": "entity_not_defined_in_data_contract", + "name": "my rule", + "expression": "a < b", + "original_entity_override": "entity_defined_in_data_contract" + } + ] + ``` diff --git a/zensical.toml b/zensical.toml index 255a771..064cb26 100644 --- a/zensical.toml +++ b/zensical.toml @@ -67,7 +67,8 @@ nav = [ {"Implementing a new backend" = "advanced_guidance/new_backend.md"}, {"Dischema Language Server" = "advanced_guidance/json_schemas.md"}, ]}, - ]} + ]}, + {"FAQ" = "faq.md"} ] extra_css = ["assets/stylesheets/extra.css"] # extra_javascript = ["assets/javascript/extra.js"] From ee0452ece9934b6be84ae62a422c4ca6a7c1e83e Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:01:57 +0100 Subject: [PATCH 08/11] docs: fix python version constraint for v0.9.0 release --- docs/user_guidance/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md index 5ad4d50..85186cd 100644 --- a/docs/user_guidance/install.md +++ b/docs/user_guidance/install.md @@ -80,7 +80,7 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru | DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | | ------------ | -------------- | -------------- | --------------- | ---------------- | -| >=0.9.0 | >=3.10,<3.12 | 1.1.3 | >=3.5.0,<=3.5.5 | 2.13.4 | +| >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | 2.13.4 | | >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 | | >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 | | >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 | From 665abee1d10c982466d29373636baaadda10efa5 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:19:20 +0100 Subject: [PATCH 09/11] docs(jsonschema): add error_details json schema and docs around them --- docs/advanced_guidance/json_schemas.md | 6 +++ .../contract/contract.schema.json | 4 ++ .../contract/error_details.schema.json | 53 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 docs/advanced_guidance/json_schemas/contract/error_details.schema.json diff --git a/docs/advanced_guidance/json_schemas.md b/docs/advanced_guidance/json_schemas.md index 5f24bd4..1c65088 100644 --- a/docs/advanced_guidance/json_schemas.md +++ b/docs/advanced_guidance/json_schemas.md @@ -17,6 +17,12 @@ For autocomplete support in VS Code, you can alter the `.vscode/settings.json` a ], "url": "./json_schemas/dataset.schema.json" }, + { + "fileMatch": [ + "*_data_contract_error_details.json" + ], + "url": "./json_schemas/contract/error_details.schema.json" + }, { "fileMatch": [ "*.rulestore.json", diff --git a/docs/advanced_guidance/json_schemas/contract/contract.schema.json b/docs/advanced_guidance/json_schemas/contract/contract.schema.json index 97c6162..dad5a92 100644 --- a/docs/advanced_guidance/json_schemas/contract/contract.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/contract.schema.json @@ -25,6 +25,10 @@ "additionalProperties": { "$ref": "components/readable_entity.schema.json" } + }, + "error_details": { + "description": "Relative path from the dischema document to reference a error details document.", + "type": "string" } }, "additionalProperties": true diff --git a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json new file mode 100644 index 0000000..58c369b --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "data-ingest:error_details.schema.json", + "title": "Data Contract Error Details", + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "description": "Name of the field defined within the data contract entities.", + "properties": { + "Bad value": { + "$ref": "#/$defs/errorObject", + "description": "When a value is incompatiable. e.g. trying to cast 'hello' as int." + }, + "Blank": { + "$ref": "#/$defs/errorObject", + "description": "When a mandatory value is missing." + } + } + } + }, + "$defs": { + "errorObject": { + "type": "object", + "properties": { + "error_code": { + "type": "string", + "description": "The code you want to report in the error report." + }, + "error_message": { + "type": "string", + "description": "The error message you want to report in the error report." + }, + "error_level": { + "type": "string", + "description": "Whether the error should be a record rejection or file/submission rejection. Defaulted to record.", + "enum": [ + "record", + "submission" + ] + }, + "is_informational": { + "type": "boolean", + "description": "Whether to raise the error as a warning or not. Defaulted to false." + }, + "reporting_entity": { + "type": "string", + "description": "Whether to rename the entity shown in the error report. Defaults to the entity name in the dischema." + } + } + } + } +} \ No newline at end of file From c03095e0e5c8a38afcf8b1de14810889bf2b6a13 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 4 Aug 2026 09:47:40 +0100 Subject: [PATCH 10/11] docs: add extra info on readme regarding functionality --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7f1b5fc..e94b29e 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ __The DVE offers__: - Format normalization to Parquet for a unified data representation - Data modelling and typecasting - Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends +- Deriving new fields and entities - Clear validation reporting, including summary insights and record-level error messages As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` (data ingest schema) file and example files can be accessed [here ↗️](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/). From db68c54204be5b4a80074d84555d0b26d9e290b0 Mon Sep 17 00:00:00 2001 From: George Robertson <50412379+georgeRobertson@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:02:41 +0100 Subject: [PATCH 11/11] docs: refactored and updated the json schemas (#131) --- .../components/base_entity.schema.json | 9 +- .../contact_error_details.schema.json | 2 +- .../contract/components/entity.schema.json | 2 +- .../contract/components/field.schema.json | 6 +- .../components/field_error_detail.schema.json | 2 +- .../field_error_type.schema copy.json | 21 -- .../components/field_error_type.schema.json | 2 +- .../field_specification.schema.json | 256 +++++++++++++----- .../components/readable_entity.schema.json | 154 +++++++++-- .../ddb_csv_header_reader.schema.json | 22 ++ .../ddb_csv_reader.schema.json | 26 ++ .../ddb_json_reader.schema.json | 19 ++ .../ddb_xml_reader.schema.json | 12 + .../global_csv_reader_args.schema.json | 25 ++ .../global_xml_reader_args.schema.json | 59 ++++ .../polars_ddb_csv_reader.schema.json | 26 ++ .../spark_csv_reader.schema.json | 38 +++ .../spark_json_reader.schema.json | 17 ++ .../spark_xml_reader.schema.json | 43 +++ .../spark_xml_stream_reader.schema.json | 12 + .../type_constraints/alphanumeric.schema.json | 17 ++ .../type_constraints/condate.schema.json | 25 ++ .../type_constraints/condecimal.schema.json | 37 +++ .../conformatteddate.schema.json | 34 +++ .../type_constraints/conint.schema.json | 33 +++ .../type_constraints/constr.schema.json | 33 +++ .../formatteddatetime.schema.json | 21 ++ .../formattedtime.schema.json | 21 ++ .../type_constraints/identifier.schema.json | 12 + .../type_constraints/nhsnumber.schema.json | 7 + .../type_constraints/orgid.schema.json | 17 ++ .../permissivenhsno.schema.json | 19 ++ .../type_constraints/postcode.schema.json | 22 ++ .../reportingperiodend.schema.json | 22 ++ .../reportingperiodstart.schema.json | 22 ++ .../contract/components/type_name.schema.json | 28 +- .../validation_function.schema.json | 34 --- .../contract/contract.schema.json | 13 +- .../contract/error_details.schema.json | 1 + .../json_schemas/dataset.schema.json | 2 +- .../json_schemas/rule_store.schema.json | 2 +- .../components/business_filter.schema.json | 24 +- .../components/business_rule.schema.json | 38 ++- .../components/concrete_filter.schema.json | 2 +- .../components/core_filter.schema.json | 46 +++- .../components/filter.schema.json | 2 +- .../multiple_expressions.schema.json | 2 +- .../components/rule.schema.json | 2 +- .../transformations.schema.json | 54 +++- 49 files changed, 1112 insertions(+), 233 deletions(-) delete mode 100644 docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json create mode 100644 docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json delete mode 100644 docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json diff --git a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json index cff1d6d..b89291a 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/base_entity.schema.json", "title": "base_entity", "description": "A complete entity specification within data ingest", @@ -10,7 +10,8 @@ "description": "A mapping of field names to their Python types. These will either be strings representing Python types (if there are no argumements to the type), and field specification objects otherwise", "additionalProperties": { "$ref": "field.schema.json" - } + }, + "minProperties": 1 }, "aliases": { "description": "A mapping of field name to allowed field alias", @@ -26,5 +27,7 @@ "type": "string" } } - } + }, + + "required": ["fields"] } diff --git a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json index 2635ac4..a9a6a91 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/contract_error_details.schema.json", "title": "base_entity", "description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)", diff --git a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json index 24453fb..b751824 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/entity.schema.json", "title": "entity", "description": "A concrete entity specification within data ingest", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json index c8454ab..f0e7d20 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field.schema.json", "title": "field", "description": "The type of a field, specified as a bare Python type if there are no argumements, and as an object otherwise. This can also be a reference to a type defined in the dataset", @@ -9,6 +9,10 @@ }, { "$ref": "field_specification.schema.json" + }, + { + "type": "string", + "minLength": 1 } ] } diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json index 5f00ca9..1939d29 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_error_detail.schema.json", "title": "field_error_detail", "description": "The custom details to be used for a field when a validation error is raised during the data contract phase", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json deleted file mode 100644 index 694948a..0000000 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-07/schema", - "$id": "data-ingest:contract/components/field_error_type.schema.json", - "title": "field_error_detail", - "description": "The error type for a field when a validation error is raised during the data contract phase", - "type": "object", - "properties": { - "error_type": { - "description": "The type of error the details are for", - "type": "string", - "enum": [ - "Blank", - "Bad value", - "Wrong format" - ], - "additionalProperties": { - "$ref": "field_error_detail.schema.json" - } - } - } -} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json index 694948a..13f1a26 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_error_type.schema.json", "title": "field_error_detail", "description": "The error type for a field when a validation error is raised during the data contract phase", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json index 5a57f4c..5564e18 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json @@ -1,16 +1,16 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_specification.schema.json", "title": "field_specification", "description": "A complete field specification within data ingest. This is generally specified using a type, model, or callable (with optional constraints) which results in a type", "type": "object", "properties": { "human_readable_name": { - "description": "A human readable name for the field", + "description": "A human readable name for the field. Non functional.", "type": "string" }, "description": { - "description": "A human-readable description of the field's purpose", + "description": "A human-readable description of the field's purpose. Non functional.", "type": "string" }, "type": { @@ -36,108 +36,216 @@ "default": { "description": "A default value for the field, to be used if it is not provided" }, - "functions": { - "description": "Validation functions to be applied to the type", - "type": "array", - "items": { - "$ref": "validation_function.schema.json" - } - }, "examples": { - "description": "An array of example values that the field could contain", + "description": "An array of example values that the field could contain. Non functional.", "type": "array" }, "disallowed_examples": { - "description": "An array of example values that the field could not contain", + "description": "An array of example values that the field could not contain. Non functional.", "type": "array" } }, - "dependencies": { - "type": { - "allOf": [ + "allOf": [ + { + "oneOf": [ + { "required": ["type"] }, + { "required": ["model"] }, + { "required": ["callable"] } + ], + "errorMessage": "Must specify exactly one of: type, model, or callable" + }, + { + "not": { + "required": ["type", "model"] + }, + "errorMessage": "Type and model are mutually exclusive" + }, + { + "not": { + "required": ["type", "callable"] + }, + "errorMessage": "Type and callable are mutually exclusive" + }, + { + "not": { + "required": ["model", "callable"] + }, + "errorMessage": "Model and callable are mutually exclusive" + }, + { + "if": { "required": ["callable"] }, + "then": { + "required": ["constraints"], + "errorMessage": "constraints is required when using callable" + } + } + ], + "unevaluatedProperties": false, + "dependentSchemas": { + "callable": { + "oneOf": [ { - "not": { - "required": [ - "model" - ] + "properties": { + "callable": { + "const": "alphanumeric" + }, + "constraints": { + "$ref": "type_constraints/alphanumeric.schema.json" + } } }, { - "not": { - "required": [ - "callable" - ] + "properties": { + "callable": { + "const": "condate" + }, + "constraints": { + "$ref": "type_constraints/condate.schema.json" + } } }, { - "not": { - "required": [ - "constraints" - ] + "properties": { + "callable": { + "const": "condecimal" + }, + "constraints": { + "$ref": "type_constraints/condecimal.schema.json" + } } - } - ] - }, - "model": { - "allOf": [ + }, { - "not": { - "required": [ - "type" - ] + "properties": { + "callable": { + "const": "conformatteddate" + }, + "constraints": { + "$ref": "type_constraints/conformatteddate.schema.json" + } } }, { - "not": { - "required": [ - "callable" - ] + "properties": { + "callable": { + "const": "conint" + }, + "constraints": { + "$ref": "type_constraints/conint.schema.json" + } } }, { - "not": { - "required": [ - "constraints" - ] + "properties": { + "callable": { + "const": "constr" + }, + "constraints": { + "$ref": "type_constraints/constr.schema.json" + } } - } - ] - }, - "callable": { - "allOf": [ + }, + { + "properties": { + "callable": { + "const": "formatteddatetime" + }, + "constraints": { + "$ref": "type_constraints/formatteddatetime.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "formattedtime" + }, + "constraints": { + "$ref": "type_constraints/formattedtime.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "identifier" + }, + "constraints": { + "$ref": "type_constraints/identifier.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "postcode" + }, + "constraints": { + "$ref": "type_constraints/postcode.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "postcode" + }, + "constraints": { + "$ref": "type_constraints/postcode.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "nhsnumber" + }, + "constraints": { + "$ref": "type_constraints/nhsnumber.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "permissivenhsno" + }, + "constraints": { + "$ref": "type_constraints/permissivenhsno.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "orgid" + }, + "constraints": { + "$ref": "type_constraints/orgid.schema.json" + } + } + }, { - "not": { - "required": [ - "type" - ] + "properties": { + "callable": { + "const": "reportingperiodstart" + }, + "constraints": { + "$ref": "type_constraints/reportingperiodstart.schema.json" + } } }, { - "not": { - "required": [ - "model" - ] + "properties": { + "callable": { + "const": "reportingperiodend" + }, + "constraints": { + "$ref": "type_constraints/reportingperiodend.schema.json" + } } } ] } }, - "oneOf": [ - { - "required": [ - "type" - ] - }, - { - "required": [ - "model" - ] - }, - { - "required": [ - "callable" - ] - } - ], "additionalProperties": false } diff --git a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json index 3f0954e..fdf2565 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/readable_entity.schema.json", "title": "entity", "description": "A concrete entity specification within data ingest. This must be readable from file", @@ -13,32 +13,144 @@ "reader_config": { "description": "Reader configuration for each entity. File extensions should be used as keys.", "type": "object", - "patternProperties": { - "^\\..+$": { - "type": "object", - "properties": { - "reader": { - "type": "string", - "description": "The name of the reader class that should be used for the file extension." - }, - "kwargs": { - "type": "object", - "description": "Additional keyword constructor args for the reader class." - } - } + "properties": { + ".csv": { + "$ref": "#/$defs/csvReaderConfig", + "description": "Configuration for loading an CSV file.", + "required": ["reader"] + }, + ".json": { + "$ref": "#/$defs/jsonReaderConfig", + "description": "Configuration for loading an JSON file.", + "required": ["reader"] + }, + ".xml": { + "$ref": "#/$defs/xmlReaderConfig", + "description": "Configuration for loading an XML file.", + "required": ["reader"] } }, - "additionalProperties": false + "minProperties": 1 }, "key_field": { "description": "The key field that should be used for the entity.", "type": "string" } - }, - "required": [ - "fields", - "reader_config" + } + } + ], + "$defs": { + "csvReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_csv_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "PolarsToDuckDBCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/polars_ddb_csv_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBCSVRepeatingHeaderReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_csv_header_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_csv_reader.schema.json" + } + } + } + ] + }, + "jsonReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBJSONReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_json_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkJSONReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_json_reader.schema.json" + } + } + } + ] + }, + "xmlReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBXMLReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_xml_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkXMLStreamReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_xml_stream_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkXMLReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_xml_reader.schema.json" + } + } + } ] } - ] -} + }, + "required": ["fields", "reader_config"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json new file mode 100644 index 0000000..7a6cb48 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_header_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVRepeatingHeaderReader", + "description": "Arguments to contol how the DuckDB CSV Header Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "ddb_csv_reader.schema.json" + } + ], + "properties": { + "non_unique_header_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when there are more than 1 set of values for a header." + }, + "non_unique_header_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when there are more than 1 set of values for a header." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json new file mode 100644 index 0000000..8a531b5 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVReader", + "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "header": { + "type": "boolean", + "description": "First line of each file contains the column names." + }, + "delim": { + "type": "string", + "description": "Delimiter character used to separate columns within each line." + }, + "quotechar": { + "type": "string", + "description": "Character used to denote the start and end of a quoted item." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json new file mode 100644 index 0000000..7cb4576 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_json_reader.schema.json", + "title": "Keyword Arguments for DuckDBJSONReader", + "description": "Arguments to contol how the DuckDB JSON Reader interacts with a json file", + "type": "object", + "properties": { + "json_format": { + "type": "string", + "description": "The format of the json file. Defaults to 'array'.", + "enum": [ + "auto", + "unstructured", + "newline_delimited", + "array" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json new file mode 100644 index 0000000..4872335 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_xml_reader.schema.json", + "title": "Keyword Arguments for DuckDBXMLStreamReader", + "description": "Arguments to contol how the DuckDB XML Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json new file mode 100644 index 0000000..4ea82ec --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/global_csv_reader_args.schema.json", + "title": "Keyword Arguments used across all CSV readers", + "description": "Arguments present in all CSV readers available.", + "type": "object", + "properties": { + "field_check": { + "type": "boolean", + "description": "Enable to check for additional or missing fields." + }, + "field_check_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when the field check fails." + }, + "field_check_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when the field check fails." + }, + "null_empty_strings": { + "type": "boolean", + "description": "Converts empty string values into 'Null' values." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json new file mode 100644 index 0000000..d82ce72 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/global_xml_reader_args.schema.json", + "title": "Keyword Arguments used across all XML readers", + "description": "Arguments present in all XML readers.", + "type": "object", + "properties": { + "record_tag": { + "type": "string", + "description": "A required string indicating the tag of each 'record' in the XML document." + }, + "root_tag": { + "type": "string", + "description": "A string indicating the tag to find the records in within the XML document." + }, + "trim_cells": { + "type": "string", + "description": "Removes leading and trailing whitespaces." + }, + "null_values": { + "type": "array", + "description": "A container of values to replace with null if encountered in an element.", + "items": { + "type": "string" + } + }, + "sanitise_multiline": { + "type": "boolean", + "description": "Whether to sanitise (remove newlines and multiple spaces) from multiline fields" + }, + "encoding": { + "type": "string", + "description": "Encoding of the XML file. Default: `utf-8-sig`" + }, + "n_records_to_read": { + "type": "integer", + "description": "The maximum number of records to read from a document" + }, + "xsd_location": { + "type": "string", + "description": "The Relative path (to the dischema) for the XSD document." + }, + "xsd_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when the XSD check fails." + }, + "xsd_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when the XSD check fails." + }, + "rules_location": { + "type": "string", + "description": "Allows you to prefix the directory which contains the XSD document." + } + }, + "required": [ + "record_tag" + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json new file mode 100644 index 0000000..7564660 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/polars_ddb_csv_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVReader", + "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "has_header": { + "type": "boolean", + "description": "First line of each file contains the column names." + }, + "separator": { + "type": "string", + "description": "Delimiter character used to separate columns within each line." + }, + "quote_char": { + "type": "string", + "description": "Character used to denote the start and end of a quoted item." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json new file mode 100644 index 0000000..e80dd7a --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_csv_reader.schema.json", + "title": "Keyword Arguments a SparkCSVReader", + "description": "Arguments to contol how the Spark CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "delimiter": { + "type": "string", + "description": "Sets a separator for each field and value. This separator can be one or more characters." + }, + "escape_char": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quotes inside an already quoted value." + }, + "quote_char": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quoted values where the separator can be part of the value. For reading, if you would like to turn off quotations, you need to set not null but an empty string." + }, + "header": { + "type": "boolean", + "description": "For reading, uses the first line as names of columns. Note that if the given path is a RDD of Strings, this header option will remove all lines same with the header if exists. CSV built-in functions ignore this option." + }, + "multi_line": { + "type": "boolean", + "description": "Allows a row to span multiple lines, by parsing line breaks within quoted values as part of the value itself. CSV built-in functions ignore this option." + }, + "encoding": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. For reading, decodes the CSV files by the given encoding type. CSV built-in functions ignore this option." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json new file mode 100644 index 0000000..af783f6 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_json_reader.schema.json", + "title": "Keyword Arguments for SparkJSONReader", + "description": "Arguments to contol how the Spark JSON Reader interacts with a json file", + "type": "object", + "properties": { + "encoding": { + "type": "string", + "description": "For reading, allows to forcibly set one of standard basic or extended encoding for the JSON files. For example UTF-16BE, UTF-32LE." + }, + "multi_line": { + "type": "boolean", + "description": "Parse one record, which may span multiple lines, per file. JSON built-in functions ignore this option." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json new file mode 100644 index 0000000..9ab1c74 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_xml_reader.schema.json", + "title": "Keyword Arguments for SparkXMLReader", + "description": "Arguments to contol how the Spark XML Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ], + "properties": { + "sampling_ratio": { + "type": "integer", + "description": "Sampling ratio for inferring schema (0.0 ~ 1)." + }, + "exclude_attribute": { + "type": "boolean", + "description": "Whether you want to exclude attributes in elements or not." + }, + "mode": { + "type": "string", + "description": "The mode for dealing with corrupt records during parsing. Permissive = When it encounters a corrupted record, it sets all fields to null and puts the malformed string into a new field configured by columnNameOfCorruptRecord. When it encounters a field of the wrong datatype, it sets the offending field to null; DROPMALFORMED = ignores the whole corrupted records; FAILFAST = throws an exception when it meets corrupted records.", + "enum": [ + "PERMISSIVE", + "FAILFAST", + "DROPMALFORMED" + ] + }, + "infer_schema": { + "type": "boolean", + "description": "If true, attempts to infer an appropriate type for each resulting DataFrame column, like a boolean, numeric or date type. If false, all resulting columns are of string type." + }, + "ignore_namespace": { + "type": "boolean", + "description": "If true, namespaces prefixes on XML elements and attributes are ignored. Tags and would, for example, be treated as if both are just . Note that, at the moment, namespaces cannot be ignored on the rowTag element, only its children. Note that XML parsing is in general not namespace-aware even if false." + }, + "namespace": { + "type": "string", + "description": "Applies a namespace to each rowTag if required." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json new file mode 100644 index 0000000..da7cdbc --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_xml_stream_reader.schema.json", + "title": "Keyword Arguments for SparkXMLStreamReader", + "description": "Arguments to contol how the Spark XML Stream Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json new file mode 100644 index 0000000..4509b3d --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/identifier.schema.json", + "title": "Constraints for alphanumeric value", + "description": "Constaints for an Alphanumeric type", + "type": "object", + "properties": { + "min_digits": { + "type": "integer", + "description": "The minimum number of alphanumeric characters required" + }, + "max_digits": { + "type": "integer", + "description": "The maximum number of alphanumeric characters required" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json new file mode 100644 index 0000000..a25ba1c --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/condate.schema.json", + "title": "Constraint for Dates", + "description": "Constaints for an Date type", + "type": "object", + "properties": { + "gt": { + "type": "string", + "description": "Enforces date to be greater than the set value" + }, + "ge": { + "type": "string", + "description": "Enforces date to be greater than or equal to the set value" + }, + "lt": { + "type": "string", + "description": "Enforces date to be less than the set value" + }, + "le": { + "type": "string", + "description": "Enforces date to be less than or equal to the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json new file mode 100644 index 0000000..4ea552c --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/condecimal.schema.json", + "title": "Constraints for Decimals", + "description": "Constaints for an Decimal type", + "type": "object", + "properties": { + "gt": { + "type": "number", + "description": "Enforces decimal to be greater than the set value" + }, + "ge": { + "type": "number", + "description": "Enforces decimal to be greater than or equal to the set value" + }, + "lt": { + "type": "number", + "description": "Enforces decimal to be less than the set value" + }, + "le": { + "type": "number", + "description": "Enforces decimal to be less than or equal to the set value" + }, + "max_digits": { + "type": "integer", + "description": "Maximum number of digits within the decimal. it does not include a zero before the decimal point or trailing decimal zeroes" + }, + "decimal_places": { + "type": "integer", + "description": "Max number of decimal places allowed. it does not include trailing decimal zeroes" + }, + "multiple_of": { + "type": "number", + "description": "Enforces decimal to be a multiple of the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json new file mode 100644 index 0000000..40fe854 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/conformatteddate.schema.json", + "title": "Constraints for Constrained Formatted Date", + "description": "Constaints for an conformatteddate type", + "type": "object", + "properties": { + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "ge": { + "type": "string", + "description": "Enforces date to be greater than or equal to the set value" + }, + "le": { + "type": "string", + "description": "Enforces date to be less than the set value" + }, + "gt": { + "type": "string", + "description": "Enforces date to be greater than the set value" + }, + "lt": { + "type": "string", + "description": "Enforces date to be less than or equal to the set value" + } + }, + "required": ["date_format"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json new file mode 100644 index 0000000..7de2c53 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/conint.schema.json", + "title": "Constraints for Integer", + "description": "Constaints for an Integer type", + "type": "object", + "properties": { + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "gt": { + "type": "integer", + "description": "Enforces integer to be greater than the set value" + }, + "ge": { + "type": "integer", + "description": "Enforces integer to be greater than or equal to the set value" + }, + "lt": { + "type": "integer", + "description": "Enforces integer to be less than the set value" + }, + "le": { + "type": "integer", + "description": "Enforces integer to be less than or equal to the set value" + }, + "multiple_of": { + "type": "integer", + "description": "Enforces integer to be a multiple of the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json new file mode 100644 index 0000000..d9fd393 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/constr.schema.json", + "title": "Constraints for String", + "description": "Constaints for an String type", + "type": "object", + "properties": { + "strip_whitespace": { + "type": "boolean", + "description": "Removes leading and trailing whitespace" + }, + "to_lower": { + "type": "boolean", + "description": "Turns all characters to lowercase" + }, + "min_length": { + "type": "integer", + "description": "Minimum length of the byte string" + }, + "max_length": { + "type": "integer", + "description": "Maximum length of the byte string" + }, + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "regex": { + "type": "string", + "description": "Enforces string to match a specific regex pattern" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json new file mode 100644 index 0000000..35691de --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/formatteddatetime.schema.json", + "title": "Constraints for Formatted Datetime", + "description": "Constaints for an FormattedDatetime type", + "type": "object", + "properties": { + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "timezone_treatment": { + "type": "string", + "enum": [ + "forbid", + "permit", + "require" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json new file mode 100644 index 0000000..c827730 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/formattedtime.schema.json", + "title": "Constraints for Formatted Time", + "description": "Constaints for an FormattedTime type", + "type": "object", + "properties": { + "time_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "timezone_treatment": { + "type": "string", + "enum": [ + "forbid", + "permit", + "require" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json new file mode 100644 index 0000000..d8323da --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/identifier.schema.json", + "title": "Constraints for alphanumeric identifier (including fixed set of special characters)", + "description": "Constaints for an Identifer type (alphanumeric + fixed set of special chars)", + "type": "object", + "allOf": [ + { + "$ref": "alphanumeric.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json new file mode 100644 index 0000000..50272ca --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/nhsnumber.schema.json", + "title": "Constraints for NHSNumber", + "description": "Constaints for the NHSNumber type", + "type": "object" +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json new file mode 100644 index 0000000..a7b9a33 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/orgid.schema.json", + "title": "Constraints for Organisation Idenitifer such as ODS Code", + "description": "Constaints for an OrgId type", + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Enforces string to match a specific regex pattern" + }, + "strip_whitespace": { + "type": "boolean", + "description": "Removes leading and trailing whitespace" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json new file mode 100644 index 0000000..4e0eeba --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/permissivenhsno.schema.json", + "title": "Constraints for NHSNumber", + "description": "Constaints for the NHSNumber type", + "type": "object", + "allOf": [ + { + "$ref": "constr.schema.json" + } + ], + "properties": { + "warn_on_test_numbers" : { + "type": "boolean", + "description": "Whether only warning should be raised if test numbers included" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json new file mode 100644 index 0000000..31685fc --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/postcode.schema.json", + "title": "Constraints for Postcode", + "description": "Constaints for an Postcode type. Defaults to UK postcode.", + "type": "object", + "allOf": [ + { + "$ref": "constr.schema.json" + } + ], + "properties": { + "pattern": { + "type": "string", + "description": "Regex format for the postcode" + }, + "apply_normalize": { + "type": "boolean", + "description": "Whether to remove whitespace from the input value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json new file mode 100644 index 0000000..d2a7f0e --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/reportingperiodend.schema.json", + "title": "Constraints for Reporting Period End Date", + "description": "Constaints for an ReportingPeriod (end) type", + "type": "object", + "properties": { + "reporting_period_type": { + "type": "string", + "description": "Enforces date to be the start or end of a month.", + "enum": [ + "start", + "end" + ] + }, + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format" + } + }, + "required": ["reporting_period_type"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json new file mode 100644 index 0000000..f5749c2 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/reportingperiodstart.schema.json", + "title": "Constraints for Reporting Period Start Date", + "description": "Constaints for an ReportingPeriod (start) type", + "type": "object", + "properties": { + "reporting_period_type": { + "type": "string", + "description": "Enforces date to be the start or end of a month.", + "enum": [ + "start", + "end" + ] + }, + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format" + } + }, + "required": ["reporting_period_type"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json index 0f331ef..660a910 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json @@ -1,7 +1,29 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/type_name.schema.json", "title": "type_name", "description": "The name of an allowed type within data ingest. This should be a Python type", - "type": "string" -} + "type": "string", + "enum": [ + "str", + "int", + "float", + "date", + "datetime", + "constr", + "conint", + "condate", + "condecimal", + "postcode", + "nhsnumber", + "permissivenhsno", + "alphanumeric", + "identifier", + "orgid", + "formatteddatetime", + "formattedtime", + "conformatteddate", + "reportingperiodstart", + "reportingperiodend" + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json b/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json deleted file mode 100644 index 69f0f85..0000000 --- a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-07/schema", - "$id": "data-ingest:contract/components/validation_function.schema.json", - "title": "validation_function", - "description": "A validation function to be applied to a field", - "type": "object", - "properties": { - "name": { - "description": "The name of the validation function to be applied", - "type": "string" - }, - "error_type": { - "description": "The type of error to emit if the function fails. This defaults to 'record_rejection'", - "type": "string", - "enum": [ - "record_rejection", - "file_rejection", - "warning" - ] - }, - "error_message": { - "description": "The message to associate with the error, if the function fails", - "type": "string" - }, - "kwargs": { - "description": "Keyword arguments to be passed to the Python function", - "type": "object" - } - }, - "required": [ - "name" - ], - "additionalProperties": false -} diff --git a/docs/advanced_guidance/json_schemas/contract/contract.schema.json b/docs/advanced_guidance/json_schemas/contract/contract.schema.json index dad5a92..f80a6e5 100644 --- a/docs/advanced_guidance/json_schemas/contract/contract.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/contract.schema.json @@ -1,8 +1,8 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/contract.schema.json", "title": "contract", - "description": "A data contract within data ingest. This specifies the information required to read in the data", + "description": "A data contract within data ingest. This specifies the information required to model and read the data", "type": "object", "properties": { "types": { @@ -24,12 +24,15 @@ "type": "object", "additionalProperties": { "$ref": "components/readable_entity.schema.json" - } + }, + "minProperties": 1 }, "error_details": { - "description": "Relative path from the dischema document to reference a error details document.", - "type": "string" + "description": "Relative path from the dischema document to reference an error details document.", + "type": "string", + "pattern": "^[a-zA-Z0-9].*_data_contract_error_details\\.json$" } }, + "required": ["datasets"], "additionalProperties": true } diff --git a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json index 58c369b..046d751 100644 --- a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json @@ -22,6 +22,7 @@ "$defs": { "errorObject": { "type": "object", + "required": ["error_code", "error_message"], "properties": { "error_code": { "type": "string", diff --git a/docs/advanced_guidance/json_schemas/dataset.schema.json b/docs/advanced_guidance/json_schemas/dataset.schema.json index 75cf4c3..4e85011 100644 --- a/docs/advanced_guidance/json_schemas/dataset.schema.json +++ b/docs/advanced_guidance/json_schemas/dataset.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:dataset.schema.json", "title": "dataset", "description": "A full dataset for data ingest. This contains the contract and the associated transformations", diff --git a/docs/advanced_guidance/json_schemas/rule_store.schema.json b/docs/advanced_guidance/json_schemas/rule_store.schema.json index cf27308..a3822bb 100644 --- a/docs/advanced_guidance/json_schemas/rule_store.schema.json +++ b/docs/advanced_guidance/json_schemas/rule_store.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:rule_store.schema.json", "title": "rule_store", "description": "A store of rules within data ingest. Property names should match the names of the contained rules.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json index d962165..372caa5 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/business_filter.schema.json", "title": "rule", "description": "Descriptions of a 'business' filter within data ingest", @@ -10,28 +10,6 @@ }, "parameters": { "type": "object" - }, - "error_code": { - "type": "string" - }, - "error_location": { - "type": "string" - }, - "reporting_entity": { - "type": "string" - }, - "reporting_field": { - "type": [ - "array", - "string" - ], - "items": { - "type": "string" - } - }, - "reporting_field_name": { - "description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.", - "type": "string" } }, "required": [ diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json index 6aae36d..07449d0 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/business_rule.schema.json", "title": "rule", "description": "Definition of a business rule within data ingest", @@ -9,10 +9,12 @@ "description": "A templateable business rule.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "transformation" @@ -21,10 +23,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "$ref": "rule.schema.json" @@ -39,10 +43,12 @@ "description": "A templateable business filter.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "filter" @@ -51,10 +57,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "$ref": "filter.schema.json" @@ -69,10 +77,12 @@ "description": "A complex rule.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "complex_rule" @@ -81,10 +91,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "type": "object", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json index a1dfd2e..c27ff08 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/concrete_filter.schema.json", "title": "rule", "description": "Description of a filter within data ingest. This filter uses a SQL expression, and requires a name.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json index 409ee98..e24da53 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json @@ -1,18 +1,21 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/core_filter.schema.json", "title": "rule", "description": "Descriptions of a filter within data ingest, implemented using a SQL expression. This filter does not require a name, but a concrete filter does.", "type": "object", "properties": { "entity": { - "type": "string" + "type": "string", + "description": "Name of the entity to perform the rule against" }, "name": { - "type": "string" + "type": "string", + "description": "Name for the filter" }, "expression": { - "type": "string" + "type": "string", + "description": "SQL expression for the filter" }, "failure_type": { "enum": [ @@ -20,10 +23,12 @@ "submission", "integrity", "group" - ] + ], + "description": "Type of failure" }, "failure_message": { - "type": "string" + "type": "string", + "description": "Customise the error message generated" }, "is_informational": { "anyOf": [ @@ -38,7 +43,8 @@ "{{ is_informational }}" ] } - ] + ], + "description": "Whether the filter is a warning or rejection" }, "error_code": { "anyOf": [ @@ -48,13 +54,16 @@ { "type": "number" } - ] + ], + "description": "Customise the error code generated" }, "error_location": { - "type": "string" + "type": "string", + "description": "Utilised where reporting_field not defined. Should represent the field/column that triggered the rule" }, "reporting_entity": { - "type": "string" + "type": "string", + "description": "Allows customisation over the name of the entity that triggered the validation rule" }, "reporting_field": { "type": [ @@ -63,17 +72,26 @@ ], "items": { "type": "string" - } + }, + "description": "The field(s) that triggered the validation rule" }, "reporting_field_name": { "description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.", "type": "string" }, "category": { - "type": "string" + "type": "string", + "enum": [ + "Blank", + "Wrong format", + "Bad value", + "Bad file" + ], + "description": "Category of failure" }, - "other_entities": { - "type": "string" + "original_entity_override": { + "type": "string", + "description": "Allows the overriding of the entity to source the id/key field(s) from" } }, "required": [ diff --git a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json index 54809ab..598dd23 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/filter.schema.json", "title": "rule", "description": "Description of a 'real' filter within data ingest", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json index 44cba18..475117b 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/multiple_expressions.schema.json", "title": "multiple_expressions", "description": "A definition of multiple aliased expressions.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json index 2e54fad..18af5a7 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/rule.schema.json", "title": "rule", "description": "Descriptions of allowed rules within data ingest", diff --git a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json index e68e175..2c64538 100644 --- a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/transformations.schema.json", "title": "transformations", "description": "Transformations within data ingest. This specifies the information required to transform and filter the data", @@ -16,31 +16,47 @@ "oneOf": [ { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "filename" + }, "filename": { "type": "string", - "description": "The path to the reference data (as Parquet) relative to the contract" + "description": "The path to the reference data (as Parquet) relative to the contract", + "pattern": ".*\\.(?:arrow|parquet)$" } }, "required": [ + "type", "filename" ] }, { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "uri" + }, "uri": { "type": "string", - "description": "The absolute URI of the reference data (as Parquet)" + "description": "The absolute URI of the reference data (as Parquet or Arrow)", + "pattern": "^[a-zA-Z][a-zA-Z0-9+.-]*:.+\\.(?:arrow|parquet)$" } }, "required": [ + "type", "uri" ] }, { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "database" + }, "database": { "type": "string", "description": "The database the reference data is stored in.", @@ -49,10 +65,12 @@ "table_name": { "type": "string", "description": "The table name (excluding database) containing the reference data.", - "pattern": "^(?:(?refdata)_)?(?[A-Za-z0-9]+)_(?\\w+)$" + "pattern": "^\\w+$" } }, "required": [ + "type", + "database", "table_name" ] } @@ -63,7 +81,8 @@ "description": "Stores of business rules", "type": "array", "items": { - "anyOf": [ + "type": "object", + "oneOf": [ { "properties": { "store_type": { @@ -72,17 +91,29 @@ "rules": { "$ref": "../rule_store.schema.json" } - } + }, + "required": [ + "store_type", + "rules" + ], + "additionalProperties": false }, { "properties": { "store_type": { - "type": "string" + "const": "json" }, "filename": { - "type": "string" + "type": "string", + "description": "Relative path to the current dischema document containing the rulestore", + "pattern": "^.*\\.json$" } - } + }, + "required": [ + "store_type", + "filename" + ], + "additionalProperties": false } ] } @@ -94,7 +125,8 @@ "type": "object", "properties": { "rule_name": { - "type": "string" + "type": "string", + "minLength": 1 }, "parameters": { "type": "object" @@ -143,4 +175,4 @@ }, "additionalProperties": false } -} +} \ No newline at end of file