diff --git a/.mise.toml b/.mise.toml index d41748e..be3149d 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,4 +1,4 @@ [tools] python="3.12" -poetry="2.3.3" +poetry="2.4.1" java="liberica-1.8.0" diff --git a/CHANGELOG.md b/CHANGELOG.md index 690c385..630e44f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.9.0 (2026-07-16) + +### Fix + +- inherit is_mandatory with get_type_and_validators +- adjusted the `is_type_complex` function to work with subclasses of 'simple' types + +### Refactor + +- upgrade pydantic to v2 (#107) + ## v0.8.3 (2026-07-14) ### Fix diff --git a/CONTRIBUTE.md b/CONTRIBUTING.md similarity index 93% rename from CONTRIBUTE.md rename to CONTRIBUTING.md index b5d1eed..d300ec4 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ poetry install --with lint ## Installation for Development -We are utilising Poetry for build dependency management and packaging. If you're on a system that has `Make` available, you can simply run `make install` to setup a local virtual environment with all the dependencies installed (this won't install Poetry for you). +We are utilising Poetry for build dependency management and packaging. To install simply run `poetry install`. If you need to update any of the dependencies then you will need to run `poetry lock` before the install command. **Please always review the contents of the lock before installing. We have a `min-release-age` of `10` which means that packages will not be updated until they have been released for at least 10 days. ## Testing diff --git a/Makefile b/Makefile index 7f31aa7..50bf37c 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,5 @@ activate = poetry run -# dev -install: - poetry lock - poetry install --with dev - # dist wheel: poetry build -f wheel diff --git a/README.md b/README.md index 7c16bb5..e94b29e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- + Data Validation Engine

@@ -9,49 +9,27 @@ [![CI Unit Tests](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml) [![CI Formatting & Linting](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml) -The Data Validation Engine (DVE) is a configuration driven data validation library built and utilised by NHS England. Currently the package has been reverted from v1.0.0 release to a 0.x as we feel the package is not yet mature enough to be considered a 1.0.0 release. So please bear this in mind if reading through the commits and references to a v1+ release when on v0.x. +The Data Validation Engine (DVE) is a configuration-driven data validation library created and used by NHS England. It lets users define validation rules once and apply them across multiple dataset collections - supporting consistent, accurate data checks. -As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` file and example files can be accessed [here](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here](https://nhsdigital.github.io/data-validation-engine/). +__The DVE offers__: -Once a dischema file has been defined, you are ready to use the DVE. The DVE is typically orchestrated based on four key "services". These are... +- SQL configuration-based validations +- Format normalization to Parquet for a unified data representation +- Data modelling and typecasting +- Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends +- Deriving new fields and entities +- Clear validation reporting, including summary insights and record-level error messages -| | Service | Purpose | -| -- | ------- | ------- | -| 1. | File Transformation | This service will take submitted files and turn them into stringified parquet file(s) to ensure that a consistent data structure can be passed through the other services. | -| 2. | Data Contract | This service will validate and perform type casting against a stringified parquet file using [pydantic models](https://docs.pydantic.dev/1.10/). | -| 3. | Business Rules | The business rules service will perform more complex validations such as comparisons between fields and tables, aggregations, filters etc to generate new entities. | -| 4. | Error Reports | The error reports service will take all the errors raised in previous services and surface them into a readable format for a downstream users/service. Currently, this implemented to be an excel spreadsheet but could be reconfigured to meet other requirements/use cases. | - -If you'd like more detailed documentation around these services the please read the extended documentation [here](https://nhsdigital.github.io/data-validation-engine/). - -The DVE has been designed in a way that's modular and can support users who just want to utilise specific "services" from the DVE (i.e. just the file transformation + data contract). Additionally, the DVE is designed to support different backend implementations. As part of the base installation of DVE, you will find backend support for `Spark` and `DuckDB`. So, if you need a `MySQL` backend implementation, you can implement this yourself. Given our organisations requirements, it will be unlikely that we add anymore specific backend implementations into the base package beyond Spark and DuckDB. So, if you are unable to implement this yourself, I would recommend reading the guidance on [requesting new features and raising bug reports here](#requesting-new-features-and-raising-bug-reports). - -Additionally, if you'd like to contribute a new backend implementation into the base DVE package, then please look at the [Contributing](#Contributing) section. +As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` (data ingest schema) file and example files can be accessed [here ↗️](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/). ## Installation and usage -The DVE is a Python package and can be installed using package managers such as [pip](https://pypi.org/project/pip/). As of the latest release we support Python 3.10 & 3.11, with Spark v3.4 and DuckDB v1.1. In the future we will be looking to upgrade the DVE to working on a higher versions of Python, DuckDB and Spark. - -If you're planning to use the Spark backend implementation, you will also need OpenJDK 11 installed. - -Python dependencies are listed in `pyproject.toml`. - -To install the DVE package you can simply install using a package manager such as [pip](https://pypi.org/project/pip/). - -``` -pip install data-validation-engine -``` - -*Note - Only versions >=0.6.2 are available on PyPi. For older versions please install directly from the git repo or build from source.* - -Once you have installed the DVE you are ready to use it. For guidance on how to create your dischema JSON document (configuration), please read the [documentation](https://nhsdigital.github.io/data-validation-engine/). - -Version 0.0.1 does support a working Python 3.7 installation. However, we will not be supporting any issues with that version of the DVE if you choose to use it. __Use at your own risk__. +Please see the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/user_guidance/install/). ## Requesting new features and raising bug reports **Before creating new issues, please check to see if the same bug/feature has been created already. Where a duplicate is created, the ticket will be closed and referenced to an existing issue.** -If you have spotted a bug with the DVE then please raise an issue [here](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template". +If you have spotted a bug with the DVE then please raise an issue [here ↗️](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template". If you have feature request then please follow the same process whilst using the "Feature request template". @@ -63,7 +41,7 @@ Below is a list of features that we would like to implement or have been request | Uplift to Python 3.11 | 0.2.0 | Yes | | Uplift Pyspark to 3.5 | 0.8.0 | Yes | | Allow DVE to run on Python 3.12+ | 0.8.0 | Yes | -| Upgrade to Pydantic 2.0 | 0.9.0 | No | +| Upgrade to Pydantic 2.0 | 0.9.0 | Yes | | Uplift Pyspark to 4.0+ | TBA | No | | Polars upgrade to v1+ | TBA | No | | DuckDB upgrade to v1.5+ | TBA | No | @@ -73,7 +51,7 @@ Below is a list of features that we would like to implement or have been request If you are interested in getting any of the unreleased features listed above available, then please read the [Contributing](#Contributing) section and then submit us a pull request. ## Contributing -Please see guidance [here](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md). +Please see guidance [here ↗️](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md). ## Legal This codebase is released under the MIT License. This covers both the codebase and any sample code in the documentation. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..abb9914 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,23 @@ +# Security + +We take security and the protection of private data extremely seriously. If you believe you have found a vulnerability or other issue which has compromised or could compromise the security of any of our systems or private data managed by our systems, please do not hesitate to contact us using the method outlined below. + +## Table of contents + +- [Security](#security) +- [Table of contents](#table-of-contents) +- [Reporting a vulnerability](#reporting-a-vulnerability) +- [General security enquiries](#general-security-enquiries) + +## Reporting a vulnerability + +If you believe you have found a security issue in this repository, please report it using GitHub's private vulnerability reporting: + +1. [Report a vulnerability](https://github.com/NHSDigital/data-validation-engine/security/advisories/new) +2. Provide details of the issue and steps to reproduce + +This creates a private channel for discussion and allows us to coordinate a fix before any public disclosure. + +## General security enquiries + +If you have general enquiries regarding our cybersecurity, please reach out to us at [cybersecurity@nhs.net](cybersecurity@nhs.net) diff --git a/docs/advanced_guidance/json_schemas.md b/docs/advanced_guidance/json_schemas.md index 5f24bd4..1c65088 100644 --- a/docs/advanced_guidance/json_schemas.md +++ b/docs/advanced_guidance/json_schemas.md @@ -17,6 +17,12 @@ For autocomplete support in VS Code, you can alter the `.vscode/settings.json` a ], "url": "./json_schemas/dataset.schema.json" }, + { + "fileMatch": [ + "*_data_contract_error_details.json" + ], + "url": "./json_schemas/contract/error_details.schema.json" + }, { "fileMatch": [ "*.rulestore.json", diff --git a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json index cff1d6d..b89291a 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/base_entity.schema.json", "title": "base_entity", "description": "A complete entity specification within data ingest", @@ -10,7 +10,8 @@ "description": "A mapping of field names to their Python types. These will either be strings representing Python types (if there are no argumements to the type), and field specification objects otherwise", "additionalProperties": { "$ref": "field.schema.json" - } + }, + "minProperties": 1 }, "aliases": { "description": "A mapping of field name to allowed field alias", @@ -26,5 +27,7 @@ "type": "string" } } - } + }, + + "required": ["fields"] } diff --git a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json index 2635ac4..a9a6a91 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/contract_error_details.schema.json", "title": "base_entity", "description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)", diff --git a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json index 24453fb..b751824 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/entity.schema.json", "title": "entity", "description": "A concrete entity specification within data ingest", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json index c8454ab..f0e7d20 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field.schema.json", "title": "field", "description": "The type of a field, specified as a bare Python type if there are no argumements, and as an object otherwise. This can also be a reference to a type defined in the dataset", @@ -9,6 +9,10 @@ }, { "$ref": "field_specification.schema.json" + }, + { + "type": "string", + "minLength": 1 } ] } diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json index 5f00ca9..1939d29 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_error_detail.schema.json", "title": "field_error_detail", "description": "The custom details to be used for a field when a validation error is raised during the data contract phase", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json deleted file mode 100644 index 694948a..0000000 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-07/schema", - "$id": "data-ingest:contract/components/field_error_type.schema.json", - "title": "field_error_detail", - "description": "The error type for a field when a validation error is raised during the data contract phase", - "type": "object", - "properties": { - "error_type": { - "description": "The type of error the details are for", - "type": "string", - "enum": [ - "Blank", - "Bad value", - "Wrong format" - ], - "additionalProperties": { - "$ref": "field_error_detail.schema.json" - } - } - } -} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json index 694948a..13f1a26 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_error_type.schema.json", "title": "field_error_detail", "description": "The error type for a field when a validation error is raised during the data contract phase", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json index 5a57f4c..5564e18 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json @@ -1,16 +1,16 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_specification.schema.json", "title": "field_specification", "description": "A complete field specification within data ingest. This is generally specified using a type, model, or callable (with optional constraints) which results in a type", "type": "object", "properties": { "human_readable_name": { - "description": "A human readable name for the field", + "description": "A human readable name for the field. Non functional.", "type": "string" }, "description": { - "description": "A human-readable description of the field's purpose", + "description": "A human-readable description of the field's purpose. Non functional.", "type": "string" }, "type": { @@ -36,108 +36,216 @@ "default": { "description": "A default value for the field, to be used if it is not provided" }, - "functions": { - "description": "Validation functions to be applied to the type", - "type": "array", - "items": { - "$ref": "validation_function.schema.json" - } - }, "examples": { - "description": "An array of example values that the field could contain", + "description": "An array of example values that the field could contain. Non functional.", "type": "array" }, "disallowed_examples": { - "description": "An array of example values that the field could not contain", + "description": "An array of example values that the field could not contain. Non functional.", "type": "array" } }, - "dependencies": { - "type": { - "allOf": [ + "allOf": [ + { + "oneOf": [ + { "required": ["type"] }, + { "required": ["model"] }, + { "required": ["callable"] } + ], + "errorMessage": "Must specify exactly one of: type, model, or callable" + }, + { + "not": { + "required": ["type", "model"] + }, + "errorMessage": "Type and model are mutually exclusive" + }, + { + "not": { + "required": ["type", "callable"] + }, + "errorMessage": "Type and callable are mutually exclusive" + }, + { + "not": { + "required": ["model", "callable"] + }, + "errorMessage": "Model and callable are mutually exclusive" + }, + { + "if": { "required": ["callable"] }, + "then": { + "required": ["constraints"], + "errorMessage": "constraints is required when using callable" + } + } + ], + "unevaluatedProperties": false, + "dependentSchemas": { + "callable": { + "oneOf": [ { - "not": { - "required": [ - "model" - ] + "properties": { + "callable": { + "const": "alphanumeric" + }, + "constraints": { + "$ref": "type_constraints/alphanumeric.schema.json" + } } }, { - "not": { - "required": [ - "callable" - ] + "properties": { + "callable": { + "const": "condate" + }, + "constraints": { + "$ref": "type_constraints/condate.schema.json" + } } }, { - "not": { - "required": [ - "constraints" - ] + "properties": { + "callable": { + "const": "condecimal" + }, + "constraints": { + "$ref": "type_constraints/condecimal.schema.json" + } } - } - ] - }, - "model": { - "allOf": [ + }, { - "not": { - "required": [ - "type" - ] + "properties": { + "callable": { + "const": "conformatteddate" + }, + "constraints": { + "$ref": "type_constraints/conformatteddate.schema.json" + } } }, { - "not": { - "required": [ - "callable" - ] + "properties": { + "callable": { + "const": "conint" + }, + "constraints": { + "$ref": "type_constraints/conint.schema.json" + } } }, { - "not": { - "required": [ - "constraints" - ] + "properties": { + "callable": { + "const": "constr" + }, + "constraints": { + "$ref": "type_constraints/constr.schema.json" + } } - } - ] - }, - "callable": { - "allOf": [ + }, + { + "properties": { + "callable": { + "const": "formatteddatetime" + }, + "constraints": { + "$ref": "type_constraints/formatteddatetime.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "formattedtime" + }, + "constraints": { + "$ref": "type_constraints/formattedtime.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "identifier" + }, + "constraints": { + "$ref": "type_constraints/identifier.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "postcode" + }, + "constraints": { + "$ref": "type_constraints/postcode.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "postcode" + }, + "constraints": { + "$ref": "type_constraints/postcode.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "nhsnumber" + }, + "constraints": { + "$ref": "type_constraints/nhsnumber.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "permissivenhsno" + }, + "constraints": { + "$ref": "type_constraints/permissivenhsno.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "orgid" + }, + "constraints": { + "$ref": "type_constraints/orgid.schema.json" + } + } + }, { - "not": { - "required": [ - "type" - ] + "properties": { + "callable": { + "const": "reportingperiodstart" + }, + "constraints": { + "$ref": "type_constraints/reportingperiodstart.schema.json" + } } }, { - "not": { - "required": [ - "model" - ] + "properties": { + "callable": { + "const": "reportingperiodend" + }, + "constraints": { + "$ref": "type_constraints/reportingperiodend.schema.json" + } } } ] } }, - "oneOf": [ - { - "required": [ - "type" - ] - }, - { - "required": [ - "model" - ] - }, - { - "required": [ - "callable" - ] - } - ], "additionalProperties": false } diff --git a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json index 3f0954e..fdf2565 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/readable_entity.schema.json", "title": "entity", "description": "A concrete entity specification within data ingest. This must be readable from file", @@ -13,32 +13,144 @@ "reader_config": { "description": "Reader configuration for each entity. File extensions should be used as keys.", "type": "object", - "patternProperties": { - "^\\..+$": { - "type": "object", - "properties": { - "reader": { - "type": "string", - "description": "The name of the reader class that should be used for the file extension." - }, - "kwargs": { - "type": "object", - "description": "Additional keyword constructor args for the reader class." - } - } + "properties": { + ".csv": { + "$ref": "#/$defs/csvReaderConfig", + "description": "Configuration for loading an CSV file.", + "required": ["reader"] + }, + ".json": { + "$ref": "#/$defs/jsonReaderConfig", + "description": "Configuration for loading an JSON file.", + "required": ["reader"] + }, + ".xml": { + "$ref": "#/$defs/xmlReaderConfig", + "description": "Configuration for loading an XML file.", + "required": ["reader"] } }, - "additionalProperties": false + "minProperties": 1 }, "key_field": { "description": "The key field that should be used for the entity.", "type": "string" } - }, - "required": [ - "fields", - "reader_config" + } + } + ], + "$defs": { + "csvReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_csv_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "PolarsToDuckDBCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/polars_ddb_csv_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBCSVRepeatingHeaderReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_csv_header_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_csv_reader.schema.json" + } + } + } + ] + }, + "jsonReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBJSONReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_json_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkJSONReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_json_reader.schema.json" + } + } + } + ] + }, + "xmlReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBXMLReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_xml_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkXMLStreamReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_xml_stream_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkXMLReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_xml_reader.schema.json" + } + } + } ] } - ] -} + }, + "required": ["fields", "reader_config"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json new file mode 100644 index 0000000..7a6cb48 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_header_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVRepeatingHeaderReader", + "description": "Arguments to contol how the DuckDB CSV Header Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "ddb_csv_reader.schema.json" + } + ], + "properties": { + "non_unique_header_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when there are more than 1 set of values for a header." + }, + "non_unique_header_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when there are more than 1 set of values for a header." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json new file mode 100644 index 0000000..8a531b5 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVReader", + "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "header": { + "type": "boolean", + "description": "First line of each file contains the column names." + }, + "delim": { + "type": "string", + "description": "Delimiter character used to separate columns within each line." + }, + "quotechar": { + "type": "string", + "description": "Character used to denote the start and end of a quoted item." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json new file mode 100644 index 0000000..7cb4576 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_json_reader.schema.json", + "title": "Keyword Arguments for DuckDBJSONReader", + "description": "Arguments to contol how the DuckDB JSON Reader interacts with a json file", + "type": "object", + "properties": { + "json_format": { + "type": "string", + "description": "The format of the json file. Defaults to 'array'.", + "enum": [ + "auto", + "unstructured", + "newline_delimited", + "array" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json new file mode 100644 index 0000000..4872335 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_xml_reader.schema.json", + "title": "Keyword Arguments for DuckDBXMLStreamReader", + "description": "Arguments to contol how the DuckDB XML Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json new file mode 100644 index 0000000..4ea82ec --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/global_csv_reader_args.schema.json", + "title": "Keyword Arguments used across all CSV readers", + "description": "Arguments present in all CSV readers available.", + "type": "object", + "properties": { + "field_check": { + "type": "boolean", + "description": "Enable to check for additional or missing fields." + }, + "field_check_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when the field check fails." + }, + "field_check_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when the field check fails." + }, + "null_empty_strings": { + "type": "boolean", + "description": "Converts empty string values into 'Null' values." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json new file mode 100644 index 0000000..d82ce72 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/global_xml_reader_args.schema.json", + "title": "Keyword Arguments used across all XML readers", + "description": "Arguments present in all XML readers.", + "type": "object", + "properties": { + "record_tag": { + "type": "string", + "description": "A required string indicating the tag of each 'record' in the XML document." + }, + "root_tag": { + "type": "string", + "description": "A string indicating the tag to find the records in within the XML document." + }, + "trim_cells": { + "type": "string", + "description": "Removes leading and trailing whitespaces." + }, + "null_values": { + "type": "array", + "description": "A container of values to replace with null if encountered in an element.", + "items": { + "type": "string" + } + }, + "sanitise_multiline": { + "type": "boolean", + "description": "Whether to sanitise (remove newlines and multiple spaces) from multiline fields" + }, + "encoding": { + "type": "string", + "description": "Encoding of the XML file. Default: `utf-8-sig`" + }, + "n_records_to_read": { + "type": "integer", + "description": "The maximum number of records to read from a document" + }, + "xsd_location": { + "type": "string", + "description": "The Relative path (to the dischema) for the XSD document." + }, + "xsd_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when the XSD check fails." + }, + "xsd_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when the XSD check fails." + }, + "rules_location": { + "type": "string", + "description": "Allows you to prefix the directory which contains the XSD document." + } + }, + "required": [ + "record_tag" + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json new file mode 100644 index 0000000..7564660 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/polars_ddb_csv_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVReader", + "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "has_header": { + "type": "boolean", + "description": "First line of each file contains the column names." + }, + "separator": { + "type": "string", + "description": "Delimiter character used to separate columns within each line." + }, + "quote_char": { + "type": "string", + "description": "Character used to denote the start and end of a quoted item." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json new file mode 100644 index 0000000..e80dd7a --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_csv_reader.schema.json", + "title": "Keyword Arguments a SparkCSVReader", + "description": "Arguments to contol how the Spark CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "delimiter": { + "type": "string", + "description": "Sets a separator for each field and value. This separator can be one or more characters." + }, + "escape_char": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quotes inside an already quoted value." + }, + "quote_char": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quoted values where the separator can be part of the value. For reading, if you would like to turn off quotations, you need to set not null but an empty string." + }, + "header": { + "type": "boolean", + "description": "For reading, uses the first line as names of columns. Note that if the given path is a RDD of Strings, this header option will remove all lines same with the header if exists. CSV built-in functions ignore this option." + }, + "multi_line": { + "type": "boolean", + "description": "Allows a row to span multiple lines, by parsing line breaks within quoted values as part of the value itself. CSV built-in functions ignore this option." + }, + "encoding": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. For reading, decodes the CSV files by the given encoding type. CSV built-in functions ignore this option." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json new file mode 100644 index 0000000..af783f6 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_json_reader.schema.json", + "title": "Keyword Arguments for SparkJSONReader", + "description": "Arguments to contol how the Spark JSON Reader interacts with a json file", + "type": "object", + "properties": { + "encoding": { + "type": "string", + "description": "For reading, allows to forcibly set one of standard basic or extended encoding for the JSON files. For example UTF-16BE, UTF-32LE." + }, + "multi_line": { + "type": "boolean", + "description": "Parse one record, which may span multiple lines, per file. JSON built-in functions ignore this option." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json new file mode 100644 index 0000000..9ab1c74 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_xml_reader.schema.json", + "title": "Keyword Arguments for SparkXMLReader", + "description": "Arguments to contol how the Spark XML Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ], + "properties": { + "sampling_ratio": { + "type": "integer", + "description": "Sampling ratio for inferring schema (0.0 ~ 1)." + }, + "exclude_attribute": { + "type": "boolean", + "description": "Whether you want to exclude attributes in elements or not." + }, + "mode": { + "type": "string", + "description": "The mode for dealing with corrupt records during parsing. Permissive = When it encounters a corrupted record, it sets all fields to null and puts the malformed string into a new field configured by columnNameOfCorruptRecord. When it encounters a field of the wrong datatype, it sets the offending field to null; DROPMALFORMED = ignores the whole corrupted records; FAILFAST = throws an exception when it meets corrupted records.", + "enum": [ + "PERMISSIVE", + "FAILFAST", + "DROPMALFORMED" + ] + }, + "infer_schema": { + "type": "boolean", + "description": "If true, attempts to infer an appropriate type for each resulting DataFrame column, like a boolean, numeric or date type. If false, all resulting columns are of string type." + }, + "ignore_namespace": { + "type": "boolean", + "description": "If true, namespaces prefixes on XML elements and attributes are ignored. Tags and would, for example, be treated as if both are just . Note that, at the moment, namespaces cannot be ignored on the rowTag element, only its children. Note that XML parsing is in general not namespace-aware even if false." + }, + "namespace": { + "type": "string", + "description": "Applies a namespace to each rowTag if required." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json new file mode 100644 index 0000000..da7cdbc --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_xml_stream_reader.schema.json", + "title": "Keyword Arguments for SparkXMLStreamReader", + "description": "Arguments to contol how the Spark XML Stream Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json new file mode 100644 index 0000000..4509b3d --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/identifier.schema.json", + "title": "Constraints for alphanumeric value", + "description": "Constaints for an Alphanumeric type", + "type": "object", + "properties": { + "min_digits": { + "type": "integer", + "description": "The minimum number of alphanumeric characters required" + }, + "max_digits": { + "type": "integer", + "description": "The maximum number of alphanumeric characters required" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json new file mode 100644 index 0000000..a25ba1c --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/condate.schema.json", + "title": "Constraint for Dates", + "description": "Constaints for an Date type", + "type": "object", + "properties": { + "gt": { + "type": "string", + "description": "Enforces date to be greater than the set value" + }, + "ge": { + "type": "string", + "description": "Enforces date to be greater than or equal to the set value" + }, + "lt": { + "type": "string", + "description": "Enforces date to be less than the set value" + }, + "le": { + "type": "string", + "description": "Enforces date to be less than or equal to the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json new file mode 100644 index 0000000..4ea552c --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/condecimal.schema.json", + "title": "Constraints for Decimals", + "description": "Constaints for an Decimal type", + "type": "object", + "properties": { + "gt": { + "type": "number", + "description": "Enforces decimal to be greater than the set value" + }, + "ge": { + "type": "number", + "description": "Enforces decimal to be greater than or equal to the set value" + }, + "lt": { + "type": "number", + "description": "Enforces decimal to be less than the set value" + }, + "le": { + "type": "number", + "description": "Enforces decimal to be less than or equal to the set value" + }, + "max_digits": { + "type": "integer", + "description": "Maximum number of digits within the decimal. it does not include a zero before the decimal point or trailing decimal zeroes" + }, + "decimal_places": { + "type": "integer", + "description": "Max number of decimal places allowed. it does not include trailing decimal zeroes" + }, + "multiple_of": { + "type": "number", + "description": "Enforces decimal to be a multiple of the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json new file mode 100644 index 0000000..40fe854 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/conformatteddate.schema.json", + "title": "Constraints for Constrained Formatted Date", + "description": "Constaints for an conformatteddate type", + "type": "object", + "properties": { + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "ge": { + "type": "string", + "description": "Enforces date to be greater than or equal to the set value" + }, + "le": { + "type": "string", + "description": "Enforces date to be less than the set value" + }, + "gt": { + "type": "string", + "description": "Enforces date to be greater than the set value" + }, + "lt": { + "type": "string", + "description": "Enforces date to be less than or equal to the set value" + } + }, + "required": ["date_format"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json new file mode 100644 index 0000000..7de2c53 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/conint.schema.json", + "title": "Constraints for Integer", + "description": "Constaints for an Integer type", + "type": "object", + "properties": { + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "gt": { + "type": "integer", + "description": "Enforces integer to be greater than the set value" + }, + "ge": { + "type": "integer", + "description": "Enforces integer to be greater than or equal to the set value" + }, + "lt": { + "type": "integer", + "description": "Enforces integer to be less than the set value" + }, + "le": { + "type": "integer", + "description": "Enforces integer to be less than or equal to the set value" + }, + "multiple_of": { + "type": "integer", + "description": "Enforces integer to be a multiple of the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json new file mode 100644 index 0000000..d9fd393 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/constr.schema.json", + "title": "Constraints for String", + "description": "Constaints for an String type", + "type": "object", + "properties": { + "strip_whitespace": { + "type": "boolean", + "description": "Removes leading and trailing whitespace" + }, + "to_lower": { + "type": "boolean", + "description": "Turns all characters to lowercase" + }, + "min_length": { + "type": "integer", + "description": "Minimum length of the byte string" + }, + "max_length": { + "type": "integer", + "description": "Maximum length of the byte string" + }, + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "regex": { + "type": "string", + "description": "Enforces string to match a specific regex pattern" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json new file mode 100644 index 0000000..35691de --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/formatteddatetime.schema.json", + "title": "Constraints for Formatted Datetime", + "description": "Constaints for an FormattedDatetime type", + "type": "object", + "properties": { + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "timezone_treatment": { + "type": "string", + "enum": [ + "forbid", + "permit", + "require" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json new file mode 100644 index 0000000..c827730 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/formattedtime.schema.json", + "title": "Constraints for Formatted Time", + "description": "Constaints for an FormattedTime type", + "type": "object", + "properties": { + "time_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "timezone_treatment": { + "type": "string", + "enum": [ + "forbid", + "permit", + "require" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json new file mode 100644 index 0000000..d8323da --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/identifier.schema.json", + "title": "Constraints for alphanumeric identifier (including fixed set of special characters)", + "description": "Constaints for an Identifer type (alphanumeric + fixed set of special chars)", + "type": "object", + "allOf": [ + { + "$ref": "alphanumeric.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json new file mode 100644 index 0000000..50272ca --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/nhsnumber.schema.json", + "title": "Constraints for NHSNumber", + "description": "Constaints for the NHSNumber type", + "type": "object" +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json new file mode 100644 index 0000000..a7b9a33 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/orgid.schema.json", + "title": "Constraints for Organisation Idenitifer such as ODS Code", + "description": "Constaints for an OrgId type", + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Enforces string to match a specific regex pattern" + }, + "strip_whitespace": { + "type": "boolean", + "description": "Removes leading and trailing whitespace" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json new file mode 100644 index 0000000..4e0eeba --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/permissivenhsno.schema.json", + "title": "Constraints for NHSNumber", + "description": "Constaints for the NHSNumber type", + "type": "object", + "allOf": [ + { + "$ref": "constr.schema.json" + } + ], + "properties": { + "warn_on_test_numbers" : { + "type": "boolean", + "description": "Whether only warning should be raised if test numbers included" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json new file mode 100644 index 0000000..31685fc --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/postcode.schema.json", + "title": "Constraints for Postcode", + "description": "Constaints for an Postcode type. Defaults to UK postcode.", + "type": "object", + "allOf": [ + { + "$ref": "constr.schema.json" + } + ], + "properties": { + "pattern": { + "type": "string", + "description": "Regex format for the postcode" + }, + "apply_normalize": { + "type": "boolean", + "description": "Whether to remove whitespace from the input value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json new file mode 100644 index 0000000..d2a7f0e --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/reportingperiodend.schema.json", + "title": "Constraints for Reporting Period End Date", + "description": "Constaints for an ReportingPeriod (end) type", + "type": "object", + "properties": { + "reporting_period_type": { + "type": "string", + "description": "Enforces date to be the start or end of a month.", + "enum": [ + "start", + "end" + ] + }, + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format" + } + }, + "required": ["reporting_period_type"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json new file mode 100644 index 0000000..f5749c2 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/reportingperiodstart.schema.json", + "title": "Constraints for Reporting Period Start Date", + "description": "Constaints for an ReportingPeriod (start) type", + "type": "object", + "properties": { + "reporting_period_type": { + "type": "string", + "description": "Enforces date to be the start or end of a month.", + "enum": [ + "start", + "end" + ] + }, + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format" + } + }, + "required": ["reporting_period_type"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json index 0f331ef..660a910 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json @@ -1,7 +1,29 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/type_name.schema.json", "title": "type_name", "description": "The name of an allowed type within data ingest. This should be a Python type", - "type": "string" -} + "type": "string", + "enum": [ + "str", + "int", + "float", + "date", + "datetime", + "constr", + "conint", + "condate", + "condecimal", + "postcode", + "nhsnumber", + "permissivenhsno", + "alphanumeric", + "identifier", + "orgid", + "formatteddatetime", + "formattedtime", + "conformatteddate", + "reportingperiodstart", + "reportingperiodend" + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json b/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json deleted file mode 100644 index 69f0f85..0000000 --- a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-07/schema", - "$id": "data-ingest:contract/components/validation_function.schema.json", - "title": "validation_function", - "description": "A validation function to be applied to a field", - "type": "object", - "properties": { - "name": { - "description": "The name of the validation function to be applied", - "type": "string" - }, - "error_type": { - "description": "The type of error to emit if the function fails. This defaults to 'record_rejection'", - "type": "string", - "enum": [ - "record_rejection", - "file_rejection", - "warning" - ] - }, - "error_message": { - "description": "The message to associate with the error, if the function fails", - "type": "string" - }, - "kwargs": { - "description": "Keyword arguments to be passed to the Python function", - "type": "object" - } - }, - "required": [ - "name" - ], - "additionalProperties": false -} diff --git a/docs/advanced_guidance/json_schemas/contract/contract.schema.json b/docs/advanced_guidance/json_schemas/contract/contract.schema.json index 97c6162..f80a6e5 100644 --- a/docs/advanced_guidance/json_schemas/contract/contract.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/contract.schema.json @@ -1,8 +1,8 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/contract.schema.json", "title": "contract", - "description": "A data contract within data ingest. This specifies the information required to read in the data", + "description": "A data contract within data ingest. This specifies the information required to model and read the data", "type": "object", "properties": { "types": { @@ -24,8 +24,15 @@ "type": "object", "additionalProperties": { "$ref": "components/readable_entity.schema.json" - } + }, + "minProperties": 1 + }, + "error_details": { + "description": "Relative path from the dischema document to reference an error details document.", + "type": "string", + "pattern": "^[a-zA-Z0-9].*_data_contract_error_details\\.json$" } }, + "required": ["datasets"], "additionalProperties": true } diff --git a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json new file mode 100644 index 0000000..046d751 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "data-ingest:error_details.schema.json", + "title": "Data Contract Error Details", + "type": "object", + "patternProperties": { + ".*": { + "type": "object", + "description": "Name of the field defined within the data contract entities.", + "properties": { + "Bad value": { + "$ref": "#/$defs/errorObject", + "description": "When a value is incompatiable. e.g. trying to cast 'hello' as int." + }, + "Blank": { + "$ref": "#/$defs/errorObject", + "description": "When a mandatory value is missing." + } + } + } + }, + "$defs": { + "errorObject": { + "type": "object", + "required": ["error_code", "error_message"], + "properties": { + "error_code": { + "type": "string", + "description": "The code you want to report in the error report." + }, + "error_message": { + "type": "string", + "description": "The error message you want to report in the error report." + }, + "error_level": { + "type": "string", + "description": "Whether the error should be a record rejection or file/submission rejection. Defaulted to record.", + "enum": [ + "record", + "submission" + ] + }, + "is_informational": { + "type": "boolean", + "description": "Whether to raise the error as a warning or not. Defaulted to false." + }, + "reporting_entity": { + "type": "string", + "description": "Whether to rename the entity shown in the error report. Defaults to the entity name in the dischema." + } + } + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/dataset.schema.json b/docs/advanced_guidance/json_schemas/dataset.schema.json index 75cf4c3..4e85011 100644 --- a/docs/advanced_guidance/json_schemas/dataset.schema.json +++ b/docs/advanced_guidance/json_schemas/dataset.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:dataset.schema.json", "title": "dataset", "description": "A full dataset for data ingest. This contains the contract and the associated transformations", diff --git a/docs/advanced_guidance/json_schemas/rule_store.schema.json b/docs/advanced_guidance/json_schemas/rule_store.schema.json index cf27308..a3822bb 100644 --- a/docs/advanced_guidance/json_schemas/rule_store.schema.json +++ b/docs/advanced_guidance/json_schemas/rule_store.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:rule_store.schema.json", "title": "rule_store", "description": "A store of rules within data ingest. Property names should match the names of the contained rules.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json index d962165..372caa5 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/business_filter.schema.json", "title": "rule", "description": "Descriptions of a 'business' filter within data ingest", @@ -10,28 +10,6 @@ }, "parameters": { "type": "object" - }, - "error_code": { - "type": "string" - }, - "error_location": { - "type": "string" - }, - "reporting_entity": { - "type": "string" - }, - "reporting_field": { - "type": [ - "array", - "string" - ], - "items": { - "type": "string" - } - }, - "reporting_field_name": { - "description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.", - "type": "string" } }, "required": [ diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json index 6aae36d..07449d0 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/business_rule.schema.json", "title": "rule", "description": "Definition of a business rule within data ingest", @@ -9,10 +9,12 @@ "description": "A templateable business rule.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "transformation" @@ -21,10 +23,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "$ref": "rule.schema.json" @@ -39,10 +43,12 @@ "description": "A templateable business filter.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "filter" @@ -51,10 +57,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "$ref": "filter.schema.json" @@ -69,10 +77,12 @@ "description": "A complex rule.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "complex_rule" @@ -81,10 +91,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "type": "object", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json index a1dfd2e..c27ff08 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/concrete_filter.schema.json", "title": "rule", "description": "Description of a filter within data ingest. This filter uses a SQL expression, and requires a name.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json index 409ee98..e24da53 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json @@ -1,18 +1,21 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/core_filter.schema.json", "title": "rule", "description": "Descriptions of a filter within data ingest, implemented using a SQL expression. This filter does not require a name, but a concrete filter does.", "type": "object", "properties": { "entity": { - "type": "string" + "type": "string", + "description": "Name of the entity to perform the rule against" }, "name": { - "type": "string" + "type": "string", + "description": "Name for the filter" }, "expression": { - "type": "string" + "type": "string", + "description": "SQL expression for the filter" }, "failure_type": { "enum": [ @@ -20,10 +23,12 @@ "submission", "integrity", "group" - ] + ], + "description": "Type of failure" }, "failure_message": { - "type": "string" + "type": "string", + "description": "Customise the error message generated" }, "is_informational": { "anyOf": [ @@ -38,7 +43,8 @@ "{{ is_informational }}" ] } - ] + ], + "description": "Whether the filter is a warning or rejection" }, "error_code": { "anyOf": [ @@ -48,13 +54,16 @@ { "type": "number" } - ] + ], + "description": "Customise the error code generated" }, "error_location": { - "type": "string" + "type": "string", + "description": "Utilised where reporting_field not defined. Should represent the field/column that triggered the rule" }, "reporting_entity": { - "type": "string" + "type": "string", + "description": "Allows customisation over the name of the entity that triggered the validation rule" }, "reporting_field": { "type": [ @@ -63,17 +72,26 @@ ], "items": { "type": "string" - } + }, + "description": "The field(s) that triggered the validation rule" }, "reporting_field_name": { "description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.", "type": "string" }, "category": { - "type": "string" + "type": "string", + "enum": [ + "Blank", + "Wrong format", + "Bad value", + "Bad file" + ], + "description": "Category of failure" }, - "other_entities": { - "type": "string" + "original_entity_override": { + "type": "string", + "description": "Allows the overriding of the entity to source the id/key field(s) from" } }, "required": [ diff --git a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json index 54809ab..598dd23 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/filter.schema.json", "title": "rule", "description": "Description of a 'real' filter within data ingest", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json index 44cba18..475117b 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/multiple_expressions.schema.json", "title": "multiple_expressions", "description": "A definition of multiple aliased expressions.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json index 2e54fad..18af5a7 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/rule.schema.json", "title": "rule", "description": "Descriptions of allowed rules within data ingest", diff --git a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json index e68e175..2c64538 100644 --- a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/transformations.schema.json", "title": "transformations", "description": "Transformations within data ingest. This specifies the information required to transform and filter the data", @@ -16,31 +16,47 @@ "oneOf": [ { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "filename" + }, "filename": { "type": "string", - "description": "The path to the reference data (as Parquet) relative to the contract" + "description": "The path to the reference data (as Parquet) relative to the contract", + "pattern": ".*\\.(?:arrow|parquet)$" } }, "required": [ + "type", "filename" ] }, { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "uri" + }, "uri": { "type": "string", - "description": "The absolute URI of the reference data (as Parquet)" + "description": "The absolute URI of the reference data (as Parquet or Arrow)", + "pattern": "^[a-zA-Z][a-zA-Z0-9+.-]*:.+\\.(?:arrow|parquet)$" } }, "required": [ + "type", "uri" ] }, { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "database" + }, "database": { "type": "string", "description": "The database the reference data is stored in.", @@ -49,10 +65,12 @@ "table_name": { "type": "string", "description": "The table name (excluding database) containing the reference data.", - "pattern": "^(?:(?refdata)_)?(?[A-Za-z0-9]+)_(?\\w+)$" + "pattern": "^\\w+$" } }, "required": [ + "type", + "database", "table_name" ] } @@ -63,7 +81,8 @@ "description": "Stores of business rules", "type": "array", "items": { - "anyOf": [ + "type": "object", + "oneOf": [ { "properties": { "store_type": { @@ -72,17 +91,29 @@ "rules": { "$ref": "../rule_store.schema.json" } - } + }, + "required": [ + "store_type", + "rules" + ], + "additionalProperties": false }, { "properties": { "store_type": { - "type": "string" + "const": "json" }, "filename": { - "type": "string" + "type": "string", + "description": "Relative path to the current dischema document containing the rulestore", + "pattern": "^.*\\.json$" } - } + }, + "required": [ + "store_type", + "filename" + ], + "additionalProperties": false } ] } @@ -94,7 +125,8 @@ "type": "object", "properties": { "rule_name": { - "type": "string" + "type": "string", + "minLength": 1 }, "parameters": { "type": "object" @@ -143,4 +175,4 @@ }, "additionalProperties": false } -} +} \ No newline at end of file diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..af25fbe --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,24 @@ +--- +title: Frequently Asked Questions (FAQ) +tags: + - FAQ + - Business Rules + - Error Reports +--- + +??? question "The `id` field in the error report is missing when it should be present?" + + If the `id` field isn't populated then it will be due to the fact that the key field is not present in the entity you are looking up against. The `key_fields`/`key_fields` is sourced from a dictionary lookup sourced in the data contract, so new entities defined within the business_rules won't have this lookup and it will result in the column not being populated. To get around this issue for entities without a defined `key_field` you can use the `original_entity_override` key to define the entity you want to source your `key_field`/`key_fields` from. You may find an example below. + + ??? example + + ```json + "filters": [ + { + "entity": "entity_not_defined_in_data_contract", + "name": "my rule", + "expression": "a < b", + "original_entity_override": "entity_defined_in_data_contract" + } + ] + ``` diff --git a/docs/user_guidance/data_contract.md b/docs/user_guidance/data_contract.md index 55913da..13e72ef 100644 --- a/docs/user_guidance/data_contract.md +++ b/docs/user_guidance/data_contract.md @@ -6,11 +6,7 @@ tags: - Domain Types --- -The Data Contract defines the structure (models) of your data and controls how it is typecast. We use [Pydantic](https://pydantic.dev/docs/validation/1.10/overview/) to generate and validate the models. This page is meant to give you greater details on how you should write your Data Contract. If you want a summary of how the Data Contract works, please refer to the [Getting Started](./getting_started.md#rules-configuration-introduction) page. - -!!! Note - - We plan to migrate to Pydantic v2+ in v0.9.0. This page currently reflects what is available through Pydantic v1. +The Data Contract defines the structure (models) of your data and controls how it is typecast. We use [Pydantic v2](https://pydantic.dev/docs/validation/latest/get-started/) to generate and validate the models. This page is meant to give you greater details on how you should write your Data Contract. If you want a summary of how the Data Contract works, please refer to the [Getting Started](./getting_started.md#rules-configuration-introduction) page. ## Models @@ -207,15 +203,15 @@ If you want to read more about the readers, please see the [File Transformation] Within the `fields` section of the contract you must define what data type a given field should be. Depending on how strict/lenient you want your types to be, a number of types are available to use. The types available are: - [Built-in standard library](https://docs.python.org/3.12/library/stdtypes.html) types (such as `int`, `str`, `date`) available with your version of Python installed for the DVE. -- [Pydantic v1 types](https://docs.pydantic.dev/1.10/usage/types/) +- [Pydantic v2 types](https://pydantic.dev/docs/validation/latest/concepts/types/) - [Custom Types](./data_contract.md#custom-types) - [Domain types](./data_contract.md#domain-types) ### Constraints -Given the DVE supports Pydantic types, you can use any of the [constrained types available](https://docs.pydantic.dev/1.10/usage/types/#constrained-types). The Pydantic docs will also show you what `kwarg` arguments are available for each constraint such as min/max length, regex patterns etc. +The DVE supports constraints against valid python/pydantic types. You can use the `constraints` arguments to apply a specifc constraint such as min/max length etc. -For example, if you wanted to use a `constr` type for a field, you would define it like this: +For example, if you wanted to use apply constraints to a `str` type, you may define it like this: ```json title="movies.dischema.json" { @@ -225,7 +221,7 @@ For example, if you wanted to use a `constr` type for a field, you would define "fields": { "movie_uuid": "int", "movie_name": { - "callable": "constr", + "callable": "str", "constraints": { "min_length": 1, "max_length": 20 @@ -249,7 +245,7 @@ As shown in the [Constraints](./data_contract.md#constraints) section above, you { "types": { "MyConstrainedString": { - "callable": "constr", + "callable": "str", "constraints": { "min_length": 1, "max_length": 20 @@ -334,7 +330,7 @@ To define a struct type, you would add it to the `types` section like this... "fields": { "movie_uuid": "int", "movie_name": { - "callable": "constr", + "callable": "str", "constraints": { "min_length": 1, "max_length": 20 diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md index e0c2ce4..85186cd 100644 --- a/docs/user_guidance/install.md +++ b/docs/user_guidance/install.md @@ -12,7 +12,7 @@ tags: If working on Python 3.7, the `0.1` release supports this (and only this) version of Python. However, we have not been updating that version with any bugfixes, performance improvements etc. There are also a number of vulnerable dependencies on version `0.1` release due to [Python 3.7 being depreciated](https://devguide.python.org/versions/) and a number of packages dropping support. **If you choose to install `0.1`, you accept the risks of doing so and additional support will not be provided.** -You can install the DVE package through python package managers such as [pip](https://pypi.org/project/pip/), [pipx](https://github.com/pypa/pipx), [uv](https://docs.astral.sh/uv/) and [poetry](https://python-poetry.org/). +You can install the DVE package through python package managers such as [pip](https://pypi.org/project/pip/), [uv](https://docs.astral.sh/uv/) and [poetry](https://python-poetry.org/). === "pip" @@ -20,12 +20,6 @@ You can install the DVE package through python package managers such as [pip](ht pip install data-validation-engine ``` -=== "pipx" - - ```sh - pipx install data-validation-engine - ``` - === "uv" Add to your existing `uv` project... @@ -84,10 +78,11 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru ## DVE Version Compatability Matrix -| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | -| ------------ | -------------- | -------------- | ------------- | ---------------- | -| >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 | -| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 | -| >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 | -| >=0.2,<0.6 | >=3.10,<3.12 | 1.1.0 | 3.4.4 | 1.10.15 | -| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 | +| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | +| ------------ | -------------- | -------------- | --------------- | ---------------- | +| >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | 2.13.4 | +| >=0.8.0 | >=3.10,<3.13 | 1.1.3 | 3.5.2 | 1.10.19 | +| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.16 | +| >=0.6 | >=3.10,<3.12 | 1.1.* | 3.4.* | 1.10.15 | +| >=0.2,<0.6 | >=3.10,<3.12 | 1.1.0 | 3.4.4 | 1.10.15 | +| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 | diff --git a/poetry.lock b/poetry.lock index c2674f9..1d13cce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,16 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] [[package]] name = "argcomplete" @@ -547,113 +559,97 @@ files = [ [[package]] name = "cffi" -version = "2.1.0" +version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = ">=3.10" +python-versions = ">=3.9" groups = ["dev", "test"] markers = "platform_python_implementation != \"PyPy\"" files = [ - {file = "cffi-2.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0"}, - {file = "cffi-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd"}, - {file = "cffi-2.1.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46"}, - {file = "cffi-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2"}, - {file = "cffi-2.1.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd"}, - {file = "cffi-2.1.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3"}, - {file = "cffi-2.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0"}, - {file = "cffi-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43"}, - {file = "cffi-2.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c"}, - {file = "cffi-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd"}, - {file = "cffi-2.1.0-cp310-cp310-win32.whl", hash = "sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f"}, - {file = "cffi-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da"}, - {file = "cffi-2.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc"}, - {file = "cffi-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7"}, - {file = "cffi-2.1.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93"}, - {file = "cffi-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2"}, - {file = "cffi-2.1.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c"}, - {file = "cffi-2.1.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f"}, - {file = "cffi-2.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565"}, - {file = "cffi-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c"}, - {file = "cffi-2.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02"}, - {file = "cffi-2.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e"}, - {file = "cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479"}, - {file = "cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458"}, - {file = "cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d"}, - {file = "cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f"}, - {file = "cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde"}, - {file = "cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d"}, - {file = "cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7"}, - {file = "cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b"}, - {file = "cffi-2.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7"}, - {file = "cffi-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66"}, - {file = "cffi-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe"}, - {file = "cffi-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b"}, - {file = "cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a"}, - {file = "cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384"}, - {file = "cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6"}, - {file = "cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda"}, - {file = "cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b"}, - {file = "cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a"}, - {file = "cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea"}, - {file = "cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db"}, - {file = "cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f"}, - {file = "cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d"}, - {file = "cffi-2.1.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0"}, - {file = "cffi-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224"}, - {file = "cffi-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c"}, - {file = "cffi-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a"}, - {file = "cffi-2.1.0-cp313-cp313-win32.whl", hash = "sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2"}, - {file = "cffi-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512"}, - {file = "cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f"}, - {file = "cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a"}, - {file = "cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3"}, - {file = "cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d"}, - {file = "cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac"}, - {file = "cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6"}, - {file = "cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913"}, - {file = "cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d"}, - {file = "cffi-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5"}, - {file = "cffi-2.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce"}, - {file = "cffi-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326"}, - {file = "cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd"}, - {file = "cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb"}, - {file = "cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804"}, - {file = "cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714"}, - {file = "cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376"}, - {file = "cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98"}, - {file = "cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13"}, - {file = "cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d"}, - {file = "cffi-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056"}, - {file = "cffi-2.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4"}, - {file = "cffi-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94"}, - {file = "cffi-2.1.0-cp314-cp314t-win32.whl", hash = "sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76"}, - {file = "cffi-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5"}, - {file = "cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8"}, - {file = "cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c"}, - {file = "cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001"}, - {file = "cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3"}, - {file = "cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc"}, - {file = "cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699"}, - {file = "cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022"}, - {file = "cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0"}, - {file = "cffi-2.1.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1"}, - {file = "cffi-2.1.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28"}, - {file = "cffi-2.1.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629"}, - {file = "cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6"}, - {file = "cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853"}, - {file = "cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda"}, - {file = "cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc"}, - {file = "cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca"}, - {file = "cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d"}, - {file = "cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8"}, - {file = "cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd"}, - {file = "cffi-2.1.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f"}, - {file = "cffi-2.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc"}, - {file = "cffi-2.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9"}, - {file = "cffi-2.1.0-cp315-cp315t-win32.whl", hash = "sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b"}, - {file = "cffi-2.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5"}, - {file = "cffi-2.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210"}, - {file = "cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb"}, + {file = "cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a"}, + {file = "cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743"}, + {file = "cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5"}, + {file = "cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5"}, + {file = "cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187"}, + {file = "cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18"}, + {file = "cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5"}, + {file = "cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b"}, + {file = "cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27"}, + {file = "cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75"}, + {file = "cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1"}, + {file = "cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f"}, + {file = "cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25"}, + {file = "cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4"}, + {file = "cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e"}, + {file = "cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6"}, + {file = "cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322"}, + {file = "cffi-2.0.0-cp39-cp39-win32.whl", hash = "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a"}, + {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, + {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] [package.dependencies] @@ -1244,14 +1240,14 @@ test = ["pytest (>=6)"] [[package]] name = "filelock" -version = "3.29.7" +version = "3.29.5" description = "A platform independent file lock." optional = false python-versions = ">=3.10" groups = ["dev"] files = [ - {file = "filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51"}, - {file = "filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d"}, + {file = "filelock-3.29.5-py3-none-any.whl", hash = "sha256:8af830889ba3a0ffcefbd6c7d2af8a54012058103771f2e10848222f476a1693"}, + {file = "filelock-3.29.5.tar.gz", hash = "sha256:6e6034c57a00a020e767f2614a5539863f056de7e7991d6d1473aef7ff73f156"}, ] [[package]] @@ -1398,6 +1394,106 @@ files = [ {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, ] +[[package]] +name = "librt" +version = "0.11.0" +description = "Mypyc runtime library" +optional = false +python-versions = ">=3.9" +groups = ["dev", "lint"] +files = [ + {file = "librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f"}, + {file = "librt-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ae627397a2f351560440d872d6f7c8dbb4072e57868e7b2fc5b8b430fe489d45"}, + {file = "librt-0.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc329359321b67d24efdf4bc69012b0597001649544db662c001db5a0184794c"}, + {file = "librt-0.11.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:7e82e642ab0f7608ce2fe53d76ca2280a9ee33a1b06556142c7c6fe80a86fc33"}, + {file = "librt-0.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88145c15c67731d54283d135b03244028c750cc9edc334a96a4f5950ebdb2884"}, + {file = "librt-0.11.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d36a51b3d93320b686588e27123f4995804dbf1bce81df78c02fc3c6eea9280"}, + {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3ac06a2a8b246327f11e186a53a100a4d5c7ed52346367e5ec751d51586c"}, + {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:461bbceede621f1ffb8839755f8663e886087ee7af16294cab7fb4d782c62eeb"}, + {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0cad8a4d6a8ff03c9b76f9414caccd78e7cfbc8a2e12fa334d8e1d9932753783"}, + {file = "librt-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f37aa505b3cf60701562eddb32df74b12a9e380c207fd8b06dd157a943ac7ea0"}, + {file = "librt-0.11.0-cp310-cp310-win32.whl", hash = "sha256:94663a21534637f0e787ec2a2a756022df6e5b7b2335a5cdd7d8e33d68a2af89"}, + {file = "librt-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:dec7db73758c2b54953fd8b7fe348c45188fe26b39ee18446196edd08453a5d4"}, + {file = "librt-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29"}, + {file = "librt-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9"}, + {file = "librt-0.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5"}, + {file = "librt-0.11.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5d63c855d86938d9de93e265c9bd8c705b51ec494de5738340ee93767a686e4b"}, + {file = "librt-0.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993f028be9e96a08d31df3479ac80d99be374d17f3b78e4796b3fd3c913d4e89"}, + {file = "librt-0.11.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:258d73a0aa66a055e65b2e4d1b8cdb23b9d132c5bb915d9547d804fcaed116cc"}, + {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0827efe7854718f04aaddf6496e96960a956e676fe1d0f04eb41511fd8ad06d5"}, + {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7753e57d6e12d019c0d8786f1c09c709f4c3fcc57c3887b24e36e6c06ec938b7"}, + {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:11bd19822431cc21af9f27374e7ae2e58103c7d98bda823536a6c47f6bb2bb3d"}, + {file = "librt-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:22bdf239b219d3993761a148ffa134b19e52e9989c84f845d5d7b71d70a17412"}, + {file = "librt-0.11.0-cp311-cp311-win32.whl", hash = "sha256:46c60b61e308eb535fbd6fa622b1ee1bb2815691c1ad9c98bf7b84952ec3bc8d"}, + {file = "librt-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:902e546ff044f579ff1c953ff5fce97b636fe9e3943996b2177710c6ef076f73"}, + {file = "librt-0.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:65ac3bc20f78aa0ee5ae84baa68917f89fef4af63e941084dd019a0d0e749f0c"}, + {file = "librt-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b87504f1690a23b9a2cca841191a04f83895d4fc2dd04df91d82b1a04ca2ad46"}, + {file = "librt-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40071fc5fe0ce8daa6de616702314a01e1250711682b0523d6ab8d4525910cb3"}, + {file = "librt-0.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:137e79445c896a0ea7b265f52d23954e05b64222ee1af69e2cb34219067cbb67"}, + {file = "librt-0.11.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:cca6644054e78746d8d4ef238681f9c34ff8b584fe6b988ecebb8db3b15e622a"}, + {file = "librt-0.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5b0eea49f5562861ee8d757a32ef7d559c1d35be2aaaa1ec28941d74c9ffc8a"}, + {file = "librt-0.11.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0d1029d7e1ae1a7e647ed6fb5df8c4ce2dffefb7a9f5fd1376a4554d96dac09f"}, + {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bc3ce6b33c5828d9e80592011a5c584cb2ce86edbc4088405f70da47dc1d1b3b"}, + {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:936c5995f3514a42111f20099397d8177c79b4d7e70961e396c6f5a0a3566766"}, + {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9bc0ca6ad9381cbe8e4aa6e5726e4c80c78115a6e9723c599ed1d73e092bc49d"}, + {file = "librt-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:070aa8c26c0a74774317a72df8851facc7f0f012a5b406557ac56992d92e1ec8"}, + {file = "librt-0.11.0-cp312-cp312-win32.whl", hash = "sha256:6bf14feb84b05ae945277395451998c89c54d0def4070eb5c08de544930b245a"}, + {file = "librt-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:75672f0bc524ede266287d532d7923dbce94c7514ad07627bac3d0c6d92cc4d9"}, + {file = "librt-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c"}, + {file = "librt-0.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:78dc31f7fdfe9c9d0eb0e8f42d139db230e826415bbcabd9f0e9faaaee909894"}, + {file = "librt-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fa475675db22290c3158e1d42326d0f5a65f04f44a0e68c3630a25b53560fb9c"}, + {file = "librt-0.11.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:621db29691044bdeda22e789e482e1b0f3a985d90e3426c9c6d17606416205ea"}, + {file = "librt-0.11.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:a9010e2ed5b3a9e158c5fd966b3ab7e834bb3d3aacc8f66c91dd4b57a3799230"}, + {file = "librt-0.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c39513d8b7477a2e1ed8c43fc21c524e8d5a0f8d4e8b7b074dbdbe7820a08e2"}, + {file = "librt-0.11.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7aef3cf1d5af86e770ab04bfd993dfc4ae8b8c17f66fb77dd4a7d50de7bbb1a3"}, + {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:557183ddc36babe46b27dd60facbd5adb4492181a5be887587d57cda6e092f21"}, + {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:83d3e1f72bd42f6c5c0b7daec530c3f829bd02db42c70b8ddf0c2d90a2459930"}, + {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:4ce1f21fbe589bc1afd7872dece84fb0e1144f794a288e58a10d2c54a55c43be"}, + {file = "librt-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b09f7044ea2b64c9da42fd3d335666518cfd1c6e8a182c95da73d0214b41e"}, + {file = "librt-0.11.0-cp313-cp313-win32.whl", hash = "sha256:78fddc31cd4d3caa897ad5d31f856b1faadc9474021ad6cb182b9018793e254e"}, + {file = "librt-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ca8aa88751a775870b764e93bad5135385f563cb8dcee399abf034ea4d3cb47"}, + {file = "librt-0.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:96f044bb325fd9cf1a723015638c219e9143f0dfbc0ca54c565df2b7fc748b44"}, + {file = "librt-0.11.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4a017a95e5837dc15a8c5661d60e05daa96b90908b1aa6b7acdf443cd25c8ebd"}, + {file = "librt-0.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b1ecbd9819deccc39b7542bf4d2a740d8a620694d39989e58661d3763458f8d4"}, + {file = "librt-0.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7da327dacd7be8f8ec36547373550744a3cc0e536d54665cd83f8bcd961200e8"}, + {file = "librt-0.11.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0dc56b1f8d06e60db362cc3fdae206681817f86ce4725d34511473487f12a34b"}, + {file = "librt-0.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05fb8fb2ab90e21c8d12ea240d744ad514da9baf381ebfa70d91d20d21713175"}, + {file = "librt-0.11.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae74872be221df4374d10fec61f93ed1513b9546ea84f2c0bf73ab3e9bd0b03"}, + {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32bcc918c0148eb7e3d57385125bac7e5f9e4359d05f07448b09f6f778c2f31c"}, + {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f9743fc99135d5f78d2454435615f6dec0473ca507c26ce9d92b10b562a280d3"}, + {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5ba067f4aadae8fda802d91d2124c90c42195ff32d9161d3549e6d05cfe26f96"}, + {file = "librt-0.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:de3bf945454d032f9e390b85c4072e0a0570bf825421c8be0e71209fa65e1abe"}, + {file = "librt-0.11.0-cp314-cp314-win32.whl", hash = "sha256:d2277a05f6dcb9fd13db9566aac4fabd68c3ea1ea46ee5567d4eef8efa495a2f"}, + {file = "librt-0.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ab73e8db5e3f564d812c1f5c3a175930a5f9bc96ccb5e3b22a34d7858b401cf7"}, + {file = "librt-0.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:aea3caa317752e3a466fa8af45d91ee0ea8c7fdd96e42b0a8dd9b76a7931eba1"}, + {file = "librt-0.11.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d1b36540d7aaf9b9101b3a6f376c8d8e9f7a9aec93ed05918f2c69d493ffef72"}, + {file = "librt-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:efbb343ab2ce3540f4ecbe6315d677ed70f37cd9a72b1e58066c918ca83acbaa"}, + {file = "librt-0.11.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0dd688aab3f7914d3e6e5e3554978e0383312fb8e771d84be008a35b9ee548"}, + {file = "librt-0.11.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f5fb36b8c6c63fdcbb1d526d94c0d1331610d43f4118cc1beb4efef4f3faacb2"}, + {file = "librt-0.11.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4a9a237d13addb93715b6fee74023d5ee3469b53fce527626c0e088aa585805f"}, + {file = "librt-0.11.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ddd17bd87b2c56ddd60e546a7984a2e64c4e8eab92fb4cf3830a48ad5469d51"}, + {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd43992b4473d42f12ff9e68326079f0696d9d4e6000e8f39a0238d482ba6ee2"}, + {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:f8e3e8056dd674e279741485e2e512d6e9a751c7455809d0114e6ebf8d781085"}, + {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c1f708d8ae9c56cf38a903c44297243d2ec83fd82b396b977e0144a3e76217e3"}, + {file = "librt-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0add982e0e7b9fc14cf4b33789d5f13f66581889b88c2f58099f6ce8f92617bd"}, + {file = "librt-0.11.0-cp314-cp314t-win32.whl", hash = "sha256:2b481d846ac894c4e8403c5fd0e87c5d11d6499e404b474602508a224ff531c8"}, + {file = "librt-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:28edb433edde181112a908c78907af28f964eabc15f4dd16c9d66c834302677c"}, + {file = "librt-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253"}, + {file = "librt-0.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6bd72d903911d995ab666dbd1871f8b1e80925a699af8063fbf50053329fb05f"}, + {file = "librt-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ef69ac715f3cd8e5cd252cb2aebfa72c015492aacc339d5d7bf8fef3c62c677"}, + {file = "librt-0.11.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:624a40c4a4ad7773315c287276cd024509b2c66ff5904f504bfc08d2c70293ab"}, + {file = "librt-0.11.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:41dc19fe150b69716c8ece4f76773a9e8813fe3e35e032a58b4d46423fb8d7c0"}, + {file = "librt-0.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4e8bd98ea9c47ae90b319a087ab28dac493f1ffbc1ecd1f28fcdbf3b7e1108d1"}, + {file = "librt-0.11.0-cp39-cp39-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84308fc49423ce6475d1c5d1985cd69a8ca9f0325fc7d5f81bb690a3f3625d4e"}, + {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ff0fbaf5f44a21beeb0110f2ab64f45135a9536a834b79c0d1ef018f2786bbfa"}, + {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:9c028a9442a18e266955d364ce42259136e79a7ba14d773e0d778d5f70cd56f1"}, + {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9f1692105a02bcf853f355032a5fdc5494358ef83d8fd22d16de375c85cec3f5"}, + {file = "librt-0.11.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7a80a71e1fda83cc752a9141e87aae7fef279538597564d670e9ce513f286192"}, + {file = "librt-0.11.0-cp39-cp39-win32.whl", hash = "sha256:140695816ddf3c86eb972981a26f35efd871c44b0c3aed44c8cd01749386617f"}, + {file = "librt-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:92f7ff819c197fc30473190a12c2856f325ac90aabfccbeb2072d28cc2e234e3"}, + {file = "librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1"}, +] + [[package]] name = "lxml" version = "6.1.1" @@ -1845,50 +1941,71 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] name = "mypy" -version = "1.11.2" +version = "1.20.2" description = "Optional static typing for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["dev", "lint"] files = [ - {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, - {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, - {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, - {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, - {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, - {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, - {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, - {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, - {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, - {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, - {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, - {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, - {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, - {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, - {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, - {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, - {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, - {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, - {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, - {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, - {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, - {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, - {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, - {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, - {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, - {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, - {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, + {file = "mypy-1.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cf5a4db6dca263010e2c7bff081c89383c72d187ba2cf4c44759aac970e2f0c4"}, + {file = "mypy-1.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b0e817b518bff7facd7f85ea05b643ad8bdcce684cf29784987b0a7c8e1f997"}, + {file = "mypy-1.20.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97d7b9a485b40f8ca425460e89bf1da2814625b2da627c0dcc6aa46c92631d14"}, + {file = "mypy-1.20.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e1c12f6d2db3d78b909b5f77513c11eb7f2dd2782b96a3ab6dffc7d44575c99"}, + {file = "mypy-1.20.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89dce27e142d25ffbc154c1819383b69f2e9234dc4ed4766f42e0e8cb264ab5c"}, + {file = "mypy-1.20.2-cp310-cp310-win_amd64.whl", hash = "sha256:f376e37f9bf2a946872fc5fd1199c99310748e3c26c7a26683f13f8bdb756cbd"}, + {file = "mypy-1.20.2-cp310-cp310-win_arm64.whl", hash = "sha256:6e2b469efd811707bc530fd1effef0f5d6eebcb7fe376affae69025da4b979a2"}, + {file = "mypy-1.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4077797a273e56e8843d001e9dfe4ba10e33323d6ade647ff260e5cd97d9758c"}, + {file = "mypy-1.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cdecf62abcc4292500d7858aeae87a1f8f1150f4c4dd08fb0b336ee79b2a6df3"}, + {file = "mypy-1.20.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c566c3a88b6ece59b3d70f65bedef17304f48eb52ff040a6a18214e1917b3254"}, + {file = "mypy-1.20.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0deb80d062b2479f2c87ae568f89845afc71d11bc41b04179e58165fd9f31e98"}, + {file = "mypy-1.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bba9ad231e92a3e424b3e56b65aa17704993425bba97e302c832f9466bb85bac"}, + {file = "mypy-1.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:baf593f2765fa3a6b1ef95807dbaa3d25b594f6a52adcc506a6b9cb115e1be67"}, + {file = "mypy-1.20.2-cp311-cp311-win_arm64.whl", hash = "sha256:20175a1c0f49863946ec20b7f63255768058ac4f07d2b9ded6a6b46cfb5a9100"}, + {file = "mypy-1.20.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4dbfcf869f6b0517f70cf0030ba6ea1d6645e132337a7d5204a18d8d5636c02b"}, + {file = "mypy-1.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b6481b228d072315b053210b01ac320e1be243dc17f9e5887ef167f23f5fae4"}, + {file = "mypy-1.20.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34397cdced6b90b836e38182076049fdb41424322e0b0728c946b0939ebdf9f6"}, + {file = "mypy-1.20.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5da6976f20cae27059ea8d0c86e7cef3de720e04c4bb9ee18e3690fdb792066"}, + {file = "mypy-1.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56908d7e08318d39f85b1f0c6cfd47b0cac1a130da677630dac0de3e0623e102"}, + {file = "mypy-1.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:d52ad8d78522da1d308789df651ee5379088e77c76cb1994858d40a426b343b9"}, + {file = "mypy-1.20.2-cp312-cp312-win_arm64.whl", hash = "sha256:785b08db19c9f214dc37d65f7c165d19a30fcecb48abfa30f31b01b5acaabb58"}, + {file = "mypy-1.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:edfbfca868cdd6bd8d974a60f8a3682f5565d3f5c99b327640cedd24c4264026"}, + {file = "mypy-1.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2877a02380adfcdbc69071a0f74d6e9dbbf593c0dc9d174e1f223ffd5281943"}, + {file = "mypy-1.20.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7488448de6007cd5177c6cea0517ac33b4c0f5ee9b5e9f2be51ce75511a85517"}, + {file = "mypy-1.20.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb9c2fa06887e21d6a3a868762acb82aec34e2c6fd0174064f27c93ede68ad15"}, + {file = "mypy-1.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d56a78b646f2e3daa865bc70cd5ec5a46c50045801ca8ff17a0c43abc97e3ee"}, + {file = "mypy-1.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:2a4102b03bb7481d9a91a6da8d174740c9c8c4401024684b9ca3b7cc5e49852f"}, + {file = "mypy-1.20.2-cp313-cp313-win_arm64.whl", hash = "sha256:a95a9248b0c6fd933a442c03c3b113c3b61320086b88e2c444676d3fd1ca3330"}, + {file = "mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30"}, + {file = "mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924"}, + {file = "mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb"}, + {file = "mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc"}, + {file = "mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558"}, + {file = "mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8"}, + {file = "mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3"}, + {file = "mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609"}, + {file = "mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2"}, + {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c"}, + {file = "mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744"}, + {file = "mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6"}, + {file = "mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec"}, + {file = "mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382"}, + {file = "mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563"}, + {file = "mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665"}, ] [package.dependencies] -mypy-extensions = ">=1.0.0" +librt = {version = ">=0.8.0", markers = "platform_python_implementation != \"PyPy\""} +mypy_extensions = ">=1.0.0" +pathspec = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.6.0" +typing_extensions = {version = ">=4.6.0", markers = "python_version < \"3.15\""} [package.extras] dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] install-types = ["pip"] mypyc = ["setuptools (>=50)"] +native-parser = ["ast-serialize (>=0.1.1,<1.0.0)"] reports = ["lxml"] [[package]] @@ -2140,8 +2257,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -2438,63 +2555,158 @@ files = [ [[package]] name = "pydantic" -version = "1.10.19" -description = "Data validation and settings management using python type hints" +version = "2.13.4" +description = "Data validation using Python type hints" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pydantic-1.10.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a415b9e95fa602b10808113967f72b2da8722061265d6af69268c111c254832d"}, - {file = "pydantic-1.10.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:11965f421f7eb026439d4eb7464e9182fe6d69c3d4d416e464a4485d1ba61ab6"}, - {file = "pydantic-1.10.19-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5bb81fcfc6d5bff62cd786cbd87480a11d23f16d5376ad2e057c02b3b44df96"}, - {file = "pydantic-1.10.19-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83ee8c9916689f8e6e7d90161e6663ac876be2efd32f61fdcfa3a15e87d4e413"}, - {file = "pydantic-1.10.19-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0399094464ae7f28482de22383e667625e38e1516d6b213176df1acdd0c477ea"}, - {file = "pydantic-1.10.19-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8b2cf5e26da84f2d2dee3f60a3f1782adedcee785567a19b68d0af7e1534bd1f"}, - {file = "pydantic-1.10.19-cp310-cp310-win_amd64.whl", hash = "sha256:1fc8cc264afaf47ae6a9bcbd36c018d0c6b89293835d7fb0e5e1a95898062d59"}, - {file = "pydantic-1.10.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d7a8a1dd68bac29f08f0a3147de1885f4dccec35d4ea926e6e637fac03cdb4b3"}, - {file = "pydantic-1.10.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07d00ca5ef0de65dd274005433ce2bb623730271d495a7d190a91c19c5679d34"}, - {file = "pydantic-1.10.19-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad57004e5d73aee36f1e25e4e73a4bc853b473a1c30f652dc8d86b0a987ffce3"}, - {file = "pydantic-1.10.19-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dce355fe7ae53e3090f7f5fa242423c3a7b53260747aa398b4b3aaf8b25f41c3"}, - {file = "pydantic-1.10.19-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0d32227ea9a3bf537a2273fd2fdb6d64ab4d9b83acd9e4e09310a777baaabb98"}, - {file = "pydantic-1.10.19-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e351df83d1c9cffa53d4e779009a093be70f1d5c6bb7068584086f6a19042526"}, - {file = "pydantic-1.10.19-cp311-cp311-win_amd64.whl", hash = "sha256:d8d72553d2f3f57ce547de4fa7dc8e3859927784ab2c88343f1fc1360ff17a08"}, - {file = "pydantic-1.10.19-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d5b5b7c6bafaef90cbb7dafcb225b763edd71d9e22489647ee7df49d6d341890"}, - {file = "pydantic-1.10.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:570ad0aeaf98b5e33ff41af75aba2ef6604ee25ce0431ecd734a28e74a208555"}, - {file = "pydantic-1.10.19-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0890fbd7fec9e151c7512941243d830b2d6076d5df159a2030952d480ab80a4e"}, - {file = "pydantic-1.10.19-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec5c44e6e9eac5128a9bfd21610df3b8c6b17343285cc185105686888dc81206"}, - {file = "pydantic-1.10.19-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6eb56074b11a696e0b66c7181da682e88c00e5cebe6570af8013fcae5e63e186"}, - {file = "pydantic-1.10.19-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9d7d48fbc5289efd23982a0d68e973a1f37d49064ccd36d86de4543aff21e086"}, - {file = "pydantic-1.10.19-cp312-cp312-win_amd64.whl", hash = "sha256:fd34012691fbd4e67bdf4accb1f0682342101015b78327eaae3543583fcd451e"}, - {file = "pydantic-1.10.19-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a5d5b877c7d3d9e17399571a8ab042081d22fe6904416a8b20f8af5909e6c8f"}, - {file = "pydantic-1.10.19-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c46f58ef2df958ed2ea7437a8be0897d5efe9ee480818405338c7da88186fb3"}, - {file = "pydantic-1.10.19-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d8a38a44bb6a15810084316ed69c854a7c06e0c99c5429f1d664ad52cec353c"}, - {file = "pydantic-1.10.19-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a82746c6d6e91ca17e75f7f333ed41d70fce93af520a8437821dec3ee52dfb10"}, - {file = "pydantic-1.10.19-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:566bebdbe6bc0ac593fa0f67d62febbad9f8be5433f686dc56401ba4aab034e3"}, - {file = "pydantic-1.10.19-cp37-cp37m-win_amd64.whl", hash = "sha256:22a1794e01591884741be56c6fba157c4e99dcc9244beb5a87bd4aa54b84ea8b"}, - {file = "pydantic-1.10.19-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:076c49e24b73d346c45f9282d00dbfc16eef7ae27c970583d499f11110d9e5b0"}, - {file = "pydantic-1.10.19-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d4320510682d5a6c88766b2a286d03b87bd3562bf8d78c73d63bab04b21e7b4"}, - {file = "pydantic-1.10.19-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e66aa0fa7f8aa9d0a620361834f6eb60d01d3e9cea23ca1a92cda99e6f61dac"}, - {file = "pydantic-1.10.19-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d216f8d0484d88ab72ab45d699ac669fe031275e3fa6553e3804e69485449fa0"}, - {file = "pydantic-1.10.19-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9f28a81978e936136c44e6a70c65bde7548d87f3807260f73aeffbf76fb94c2f"}, - {file = "pydantic-1.10.19-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d3449633c207ec3d2d672eedb3edbe753e29bd4e22d2e42a37a2c1406564c20f"}, - {file = "pydantic-1.10.19-cp38-cp38-win_amd64.whl", hash = "sha256:7ea24e8614f541d69ea72759ff635df0e612b7dc9d264d43f51364df310081a3"}, - {file = "pydantic-1.10.19-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:573254d844f3e64093f72fcd922561d9c5696821ff0900a0db989d8c06ab0c25"}, - {file = "pydantic-1.10.19-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ff09600cebe957ecbb4a27496fe34c1d449e7957ed20a202d5029a71a8af2e35"}, - {file = "pydantic-1.10.19-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4739c206bfb6bb2bdc78dcd40bfcebb2361add4ceac6d170e741bb914e9eff0f"}, - {file = "pydantic-1.10.19-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bfb5b378b78229119d66ced6adac2e933c67a0aa1d0a7adffbe432f3ec14ce4"}, - {file = "pydantic-1.10.19-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7f31742c95e3f9443b8c6fa07c119623e61d76603be9c0d390bcf7e888acabcb"}, - {file = "pydantic-1.10.19-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c6444368b651a14c2ce2fb22145e1496f7ab23cbdb978590d47c8d34a7bc0289"}, - {file = "pydantic-1.10.19-cp39-cp39-win_amd64.whl", hash = "sha256:945407f4d08cd12485757a281fca0e5b41408606228612f421aa4ea1b63a095d"}, - {file = "pydantic-1.10.19-py3-none-any.whl", hash = "sha256:2206a1752d9fac011e95ca83926a269fb0ef5536f7e053966d058316e24d929f"}, - {file = "pydantic-1.10.19.tar.gz", hash = "sha256:fea36c2065b7a1d28c6819cc2e93387b43dd5d3cf5a1e82d8132ee23f36d1f10"}, + {file = "pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba"}, + {file = "pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6"}, ] [package.dependencies] -typing-extensions = ">=4.2.0" +annotated-types = ">=0.6.0" +pydantic-core = "2.46.4" +typing-extensions = ">=4.14.1" +typing-inspection = ">=0.4.2" [package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4"}, + {file = "pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01"}, + {file = "pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d"}, + {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4"}, + {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f"}, + {file = "pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39"}, + {file = "pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d"}, + {file = "pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf"}, + {file = "pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594"}, + {file = "pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398"}, + {file = "pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3"}, + {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848"}, + {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3"}, + {file = "pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109"}, + {file = "pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda"}, + {file = "pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33"}, + {file = "pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d"}, + {file = "pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2"}, + {file = "pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987"}, + {file = "pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b"}, + {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458"}, + {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b"}, + {file = "pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c"}, + {file = "pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894"}, + {file = "pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89"}, + {file = "pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a"}, + {file = "pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008"}, + {file = "pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262"}, + {file = "pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e"}, + {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd"}, + {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be"}, + {file = "pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d"}, + {file = "pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb"}, + {file = "pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292"}, + {file = "pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d"}, + {file = "pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb"}, + {file = "pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c"}, + {file = "pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb"}, + {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898"}, + {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e"}, + {file = "pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519"}, + {file = "pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4"}, + {file = "pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac"}, + {file = "pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5"}, + {file = "pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596"}, + {file = "pydantic_core-2.46.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae"}, + {file = "pydantic_core-2.46.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_31_riscv64.whl", hash = "sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066"}, + {file = "pydantic_core-2.46.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6"}, + {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9"}, + {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29"}, + {file = "pydantic_core-2.46.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9"}, + {file = "pydantic_core-2.46.4-cp39-cp39-win32.whl", hash = "sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1"}, + {file = "pydantic_core-2.46.4-cp39-cp39-win_amd64.whl", hash = "sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b"}, + {file = "pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526"}, + {file = "pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc"}, + {file = "pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983"}, + {file = "pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1"}, +] + +[package.dependencies] +typing-extensions = ">=4.14.1" [[package]] name = "pygments" @@ -2528,8 +2740,8 @@ astroid = ">=3.3.8,<=3.4.0.dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.2", markers = "python_version < \"3.11\""}, - {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, {version = ">=0.3.6", markers = "python_version == \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, ] isort = ">=4.2.5,<5.13 || >5.13,<7" mccabe = ">=0.6,<0.8" @@ -3117,16 +3329,31 @@ files = [ ] markers = {docs = "python_version == \"3.10\"", test = "python_version == \"3.10\""} +[[package]] +name = "typing-inspection" +version = "0.4.2" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"}, + {file = "typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + [[package]] name = "tzdata" -version = "2026.3" +version = "2026.2" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" groups = ["main"] files = [ - {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, - {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, + {file = "tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7"}, + {file = "tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"}, ] [[package]] @@ -3425,4 +3652,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "305caceb6c037646196a148e44272bc1a39901af5027c3f27b6858bd67edff41" +content-hash = "2092f1fb683bfd95d9be38901934e860bed349a5cd8d5a51fab7c3e583c5f542" diff --git a/poetry.toml b/poetry.toml index 9255596..9d82d94 100644 --- a/poetry.toml +++ b/poetry.toml @@ -1,5 +1,8 @@ [experimental] new-installer = false +[solver] +min-release-age = 10 + [virtualenvs] in-project = true diff --git a/pyproject.toml b/pyproject.toml index 9658e6d..8e08264 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Operating System :: OS Independent", + "Operating System :: POSIX :: Linux", "Topic :: Software Development :: Libraries", "Typing :: Typed", ] @@ -25,7 +25,7 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues" Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md" [tool.poetry] -version = "0.8.3" +version = "0.9.0rc0" packages = [ { include = "dve", from = "src" }, ] @@ -43,7 +43,7 @@ openpyxl = "3.1.5" pandas = "2.3.3" polars = "0.20.31" pyarrow = "23.0.1" -pydantic = "1.10.19" +pydantic = "2.13.4" pyspark = ">=3.5.0,<=3.5.5" typing_extensions = "4.15.0" urllib3 = "2.7.0" # dependency of boto3 & botocore @@ -84,7 +84,8 @@ black = "26.3.1" astroid = "3.3.9" isort = "5.13.2" pylint = "3.3.9" -mypy = "1.11.2" +mypy = "1.20.2" +librt = "0.11.0" # mypy dependency boto3-stubs = {extras = ["essential"], version = "1.26.72"} botocore-stubs = "1.29.72" pandas-stubs = "1.2.0.62" diff --git a/src/dve/core_engine/backends/base/auditing.py b/src/dve/core_engine/backends/base/auditing.py index 3b90a4d..24cfd3b 100644 --- a/src/dve/core_engine/backends/base/auditing.py +++ b/src/dve/core_engine/backends/base/auditing.py @@ -15,7 +15,7 @@ from types import TracebackType from typing import Any, ClassVar, Generic, Optional, TypeVar, Union -from pydantic import ValidationError, validate_arguments +from pydantic import ValidationError, validate_call from typing_extensions import Literal, get_origin from dve.core_engine.models import ( @@ -99,8 +99,8 @@ def __init__(self, name: str, record_type: type[AuditRecord]): def schema(self) -> dict[str, type]: """Determine python schema of auditor""" return { - fld: str if get_origin(mdl.type_) == Literal else mdl.type_ - for fld, mdl in self._record_type.__fields__.items() + fld: str if get_origin(mdl.annotation) == Literal else mdl.annotation # type: ignore + for fld, mdl in self._record_type.model_fields.items() } @staticmethod @@ -196,7 +196,7 @@ def conv_to_iterable(recs: Union[AuditorType, AuditReturnType]) -> Iterable[dict """Convert AuditReturnType to iterable of dictionaries""" raise NotImplementedError() - @validate_arguments + @validate_call def add_processing_records(self, processing_records: list[ProcessingStatusRecord]): """Add an entry to the processing_status auditor.""" if self.pool: @@ -208,7 +208,7 @@ def add_processing_records(self, processing_records: list[ProcessingStatusRecord records=[dict(rec) for rec in processing_records] ) - @validate_arguments + @validate_call def add_submission_statistics_records(self, sub_stats: list[SubmissionStatisticsRecord]): """Add an entry to the submission statistics auditor.""" if self.pool: @@ -218,7 +218,7 @@ def add_submission_statistics_records(self, sub_stats: list[SubmissionStatistics ) return self._submission_statistics.add_records(records=[dict(rec) for rec in sub_stats]) - @validate_arguments + @validate_call def add_transfer_records(self, transfer_records: list[TransferRecord]): """Add an entry to the transfers auditor""" if self.pool: @@ -227,7 +227,7 @@ def add_transfer_records(self, transfer_records: list[TransferRecord]): ) return self._transfers.add_records(records=[dict(rec) for rec in transfer_records]) - @validate_arguments + @validate_call def add_new_submissions( self, submissions: list[SubmissionMetadata], @@ -250,7 +250,7 @@ def add_new_submissions( processing_status="received", job_run_id=job_run_id, **ts_info, - ).dict(), + ).model_dump(), } processing_status_recs.append(processing_rec) if sub_info: diff --git a/src/dve/core_engine/backends/implementations/duckdb/auditing.py b/src/dve/core_engine/backends/implementations/duckdb/auditing.py index ffb043e..803b696 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/auditing.py +++ b/src/dve/core_engine/backends/implementations/duckdb/auditing.py @@ -14,10 +14,10 @@ OrderCriteria, ) from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import ( - PYTHON_TYPE_TO_DUCKDB_TYPE, + get_duckdb_type_from_annotation, table_exists, ) -from dve.core_engine.backends.utilities import PYTHON_TYPE_TO_POLARS_TYPE +from dve.core_engine.backends.utilities import get_polars_type_from_annotation from dve.core_engine.models import ( AuditRecord, ProcessingStatusRecord, @@ -62,7 +62,10 @@ def ddb_create_table_sql(self) -> str: """Generate create table sql script for auditor""" _sql_expression = f"CREATE TABLE {self._name} (" _sql_expression += ", ".join( - [f"{fld} {PYTHON_TYPE_TO_DUCKDB_TYPE.get(dtype)}" for fld, dtype in self.schema.items()] + [ + f"{fld} {get_duckdb_type_from_annotation(dtype)}" + for fld, dtype in self.schema.items() + ] ) _sql_expression += ")" return _sql_expression @@ -70,10 +73,7 @@ def ddb_create_table_sql(self) -> str: @property def polars_schema(self) -> dict[str, PolarsType]: """Get polars dataframe schema for auditor""" - return { - fld: PYTHON_TYPE_TO_POLARS_TYPE.get(dtype, pl.Utf8) # type: ignore - for fld, dtype in self.schema.items() - } + return {fld: get_polars_type_from_annotation(dtype) for fld, dtype in self.schema.items()} def get_relation(self) -> DuckDBPyRelation: """Get a relation to interact with the auditor duckdb table""" @@ -106,7 +106,7 @@ def conv_to_entity(self, recs: list[AuditRecord]) -> DuckDBPyRelation: """Convert a list of audit records to a relation""" # pylint: disable=W0612 rec_df = pl.DataFrame( # type: ignore - [rec.dict() for rec in recs], + [rec.model_dump() for rec in recs], schema=self.polars_schema, ) return self._connection.sql("select * from rec_df") diff --git a/src/dve/core_engine/backends/implementations/duckdb/contract.py b/src/dve/core_engine/backends/implementations/duckdb/contract.py index 3595716..d9bb9bc 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/contract.py +++ b/src/dve/core_engine/backends/implementations/duckdb/contract.py @@ -14,7 +14,6 @@ from duckdb.typing import DuckDBPyType from polars.datatypes.classes import DataTypeClass as PolarsType from pydantic import BaseModel -from pydantic.fields import ModelField import dve.parser.file_handling as fh from dve.common.error_utils import ( @@ -96,8 +95,8 @@ def create_entity_from_py_iterator( # pylint: disable=unused-argument ) -> DuckDBPyRelation: """Create DuckDB Relation from iterator of records""" polars_schema: dict[str, PolarsType] = { - fld.name: get_polars_type_from_annotation(fld.type_) - for fld in stringify_model(schema).__fields__.values() + name: get_polars_type_from_annotation(fld.annotation) + for name, fld in stringify_model(schema).model_fields.items() } _lazy_df = pl.LazyFrame(records, polars_schema) # type: ignore # pylint: disable=unused-variable return self._connection.sql("select * from _lazy_df") @@ -130,17 +129,15 @@ def apply_data_contract( ) as msg_writer: for entity_name, relation in entities.items(): # get dtypes for all fields -> python data types or use with relation - entity_fields: dict[str, ModelField] = contract_metadata.schemas[ - entity_name - ].__fields__ + entity_fields = contract_metadata.schemas[entity_name].model_fields ddb_schema: dict[str, DuckDBPyType] = { - fld.name: get_duckdb_type_from_annotation(fld.annotation) - for fld in entity_fields.values() + name: get_duckdb_type_from_annotation(fld.annotation) + for name, fld in entity_fields.items() } ddb_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int) polars_schema: dict[str, PolarsType] = { - fld.name: get_polars_type_from_annotation(fld.annotation) - for fld in entity_fields.values() + name: get_polars_type_from_annotation(fld.annotation) + for name, fld in entity_fields.items() } polars_schema[RECORD_INDEX_COLUMN_NAME] = get_polars_type_from_annotation(int) if relation_is_empty(relation): diff --git a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py index b54192f..b7d87d8 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py +++ b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py @@ -8,7 +8,7 @@ from datetime import date, datetime, time from decimal import Decimal from pathlib import Path -from typing import Any, ClassVar, Union +from typing import Any, ClassVar, Literal, Union from urllib.parse import urlparse import duckdb.typing as ddbtyp @@ -128,8 +128,8 @@ def get_duckdb_type_from_annotation(type_annotation: Any) -> DuckDBPyType: 'optional' wrapper and return the inner type - A subclass of `typing.TypedDict` with values typed using supported types. This will parse the value types as Polars types and return a duckdb STRUCT. - - A dataclass or `pydantic.main.ModelMetaClass` with values typed using supported types. - This will parse the field types as Polars types and return a duckdb STRUCT. + - A dataclass or `pydantic.BaseModel` with values typed using supported types. + This will parse the field types as duckdb types and return a duckdb STRUCT. - Any supported type, with a `typing_extensions.Annotated` wrapper. Any `ClassVar` types within `TypedDict`s, dataclasses, or `pydantic` models will be @@ -138,6 +138,14 @@ def get_duckdb_type_from_annotation(type_annotation: Any) -> DuckDBPyType: """ type_origin = get_origin(type_annotation) + if type_origin is Literal: + ddb_types = [get_duckdb_type_from_annotation(type(t)) for t in get_args(type_annotation)] + if not ddb_types or not all(t == ddb_types[0] for t in ddb_types): + raise ValueError( + f"Unable to determine a single concrete type for Literal. Got {type_annotation!r}" + ) + return ddb_types[0] + # An `Optional` or `Union` type, check to ensure non-heterogenity. if type_origin is Union: python_type = _get_non_heterogenous_type(get_args(type_annotation)) diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py index 3cd931d..723e5e3 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py @@ -111,8 +111,8 @@ def read_to_relation( # pylint: disable=unused-argument } ddb_schema: dict[str, SQLType] = { - fld.name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore - for fld in schema.__fields__.values() + name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore + for name, fld in schema.model_fields.items() } reader_options["columns"] = ddb_schema @@ -166,8 +166,8 @@ def read_to_relation( # pylint: disable=unused-argument } polars_types = { - fld.name: get_polars_type_from_annotation(fld.annotation) # type: ignore - for fld in schema.__fields__.values() + name: get_polars_type_from_annotation(fld.annotation) # type: ignore + for name, fld in schema.model_fields.items() } reader_options["dtypes"] = polars_types diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py index 09c2384..79d74c6 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py @@ -56,8 +56,8 @@ def read_to_relation( # pylint: disable=unused-argument """Returns a relation object from the source json""" ddb_schema: dict[str, SQLType] = { - fld.name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore - for fld in schema.__fields__.values() + name: str(get_duckdb_type_from_annotation(fld.annotation)) # type: ignore + for name, fld in schema.model_fields.items() } return self.add_record_index( diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py index 00d81a6..42e281a 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py @@ -47,8 +47,8 @@ def read_to_relation( ) polars_schema: dict[str, pl.DataType] = { # type: ignore - fld.name: get_polars_type_from_annotation(fld.annotation) - for fld in stringify_model(schema).__fields__.values() + name: get_polars_type_from_annotation(fld.annotation) + for name, fld in stringify_model(schema).model_fields.items() } _lazy_frame = self.add_record_index( diff --git a/src/dve/core_engine/backends/implementations/spark/auditing.py b/src/dve/core_engine/backends/implementations/spark/auditing.py index c050a17..3f50721 100644 --- a/src/dve/core_engine/backends/implementations/spark/auditing.py +++ b/src/dve/core_engine/backends/implementations/spark/auditing.py @@ -116,7 +116,7 @@ def conv_to_records(self, recs: DataFrame) -> Iterable[AuditRecord]: def conv_to_entity(self, recs: list[AuditRecord]) -> DataFrame: """Convert the dataframe to an iterable of the related audit record""" return self._spark.createDataFrame( # type: ignore - [rec.dict() for rec in recs], schema=self.spark_schema + [rec.model_dump() for rec in recs], schema=self.spark_schema ) def add_records(self, records: Iterable[dict[str, Any]]): diff --git a/src/dve/core_engine/backends/implementations/spark/backend.py b/src/dve/core_engine/backends/implementations/spark/backend.py index 126e07a..abd1540 100644 --- a/src/dve/core_engine/backends/implementations/spark/backend.py +++ b/src/dve/core_engine/backends/implementations/spark/backend.py @@ -92,5 +92,5 @@ def write_entities_to_parquet( def convert_submission_info(self, submission_info: SubmissionInfo) -> DataFrame: return self.spark_session.createDataFrame( # type: ignore - [submission_info.dict()], schema=get_type_from_annotation(type(submission_info)) + [submission_info.model_dump()], schema=get_type_from_annotation(type(submission_info)) ) diff --git a/src/dve/core_engine/backends/implementations/spark/contract.py b/src/dve/core_engine/backends/implementations/spark/contract.py index e84958e..d2fd9ae 100644 --- a/src/dve/core_engine/backends/implementations/spark/contract.py +++ b/src/dve/core_engine/backends/implementations/spark/contract.py @@ -7,7 +7,7 @@ from uuid import uuid4 from pydantic import BaseModel -from pydantic.fields import ModelField +from pydantic.fields import FieldInfo from pyspark.sql import DataFrame, SparkSession from pyspark.sql import functions as sf from pyspark.sql.functions import col, lit @@ -104,7 +104,9 @@ def apply_data_contract( successful = True for entity_name, record_df in entities.items(): - entity_fields: dict[str, ModelField] = contract_metadata.schemas[entity_name].__fields__ + entity_fields: dict[str, FieldInfo] = contract_metadata.schemas[ + entity_name + ].model_fields spark_schema = get_type_from_annotation(contract_metadata.schemas[entity_name]) spark_schema.add(StructField(RECORD_INDEX_COLUMN_NAME, LongType())) if df_is_empty(record_df): @@ -147,19 +149,17 @@ def apply_data_contract( self.logger.info(f"Data contract found {msg_count} issues in {entity_name}") try: - # TODO: will need to revisit in pydantic v2 bump as model field no longer available record_df = record_df.select( *[ ( get_spark_cast_statement_from_annotation( - fld, mdl_field.annotation + fld, fld_info.annotation ).alias(fld) if fld in record_df.columns else lit(None).cast( - get_type_from_annotation(mdl_field.annotation).alias(fld) - ) + get_type_from_annotation(fld_info.annotation)).alias(fld) ) - for fld, mdl_field in entity_fields.items() + for fld, fld_info in entity_fields.items() ], col(RECORD_INDEX_COLUMN_NAME).cast(LongType()).alias(RECORD_INDEX_COLUMN_NAME), ) diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py index ae714a6..981cf13 100644 --- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py +++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py @@ -13,11 +13,10 @@ from decimal import Decimal from functools import wraps from pathlib import Path -from typing import Any, ClassVar, Optional, TypeVar, Union, overload +from typing import Any, ClassVar, Literal, Optional, TypeVar, Union, overload from delta.exceptions import ConcurrentAppendException, DeltaConcurrentModificationException from pydantic import BaseModel -from pydantic.types import ConstrainedDecimal from pyspark.sql import DataFrame, Row, SparkSession from pyspark.sql import functions as sf from pyspark.sql import types as st @@ -52,6 +51,7 @@ """A wrapped function (Spark UDF) taking four args.""" +# TODO - lets see if we can bin this off as it's a bit overkill @dataclass(frozen=True) class DecimalConfig: """Configuration for a Python decimal to enable it to be mapped to a @@ -64,13 +64,13 @@ class DecimalConfig: """ - precision: int = 38 + max_digits: int = 38 """ The precision of the decimal. This is the total number of digits in the decimal. """ - scale: int = 18 + decimal_places: int = 18 """ The scale of the decimal. This is the number of digits to the right of the decimal point. @@ -78,10 +78,12 @@ class DecimalConfig: """ def __post_init__(self): - if not 0 < self.precision <= 38: - raise ValueError("Precision must be between 1 and 38 (inclusive)") - if not 0 <= self.scale <= self.precision: - raise ValueError("Scale must be between 0 and the precision (inclusive)") + if not 0 < self.max_digits <= 38: + raise ValueError("Max digits must be between 1 and 38 (inclusive)") + if not 0 <= self.decimal_places <= self.max_digits: + raise ValueError( + "Decimal Places must be between 0 and the specified number of digits (inclusive)" + ) DEFAULT_DECIMAL_CONFIG = DecimalConfig() @@ -96,7 +98,9 @@ def __post_init__(self): bytes: st.BinaryType(), dt.date: st.DateType(), dt.datetime: st.TimestampType(), - Decimal: st.DecimalType(DEFAULT_DECIMAL_CONFIG.precision, DEFAULT_DECIMAL_CONFIG.scale), + Decimal: st.DecimalType( + DEFAULT_DECIMAL_CONFIG.max_digits, DEFAULT_DECIMAL_CONFIG.decimal_places + ), } """A mapping of Python types to the equivalent Spark types.""" @@ -164,7 +168,7 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType: 'optional' wrapper and return the inner type (Spark types are all nullable). - A subclass of `typing.TypedDict` with values typed using supported types. This will parse the value types as Spark types and return a Spark `StructType`. - - A dataclass or `pydantic.main.ModelMetaClass` with values typed using supported types. + - A dataclass or `pydantic.BaseModel` with values typed using supported types. This will parse the field types as Spark types and return a Spark `StructType`. - Any supported type, with a `typing_extensions.Annotated` wrapper. - A `decimal.Decimal` wrapped with `typing_extensions.Annotated` with a `DecimalConfig` @@ -178,6 +182,14 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType: """ type_origin = get_origin(type_annotation) + if type_origin is Literal: + types = [get_type_from_annotation(type(t)) for t in get_args(type_annotation)] + if not types or not all(t == types[0] for t in types): + raise ValueError( + f"Unable to determine a single concrete type for Literal. Got {type_annotation!r}" + ) + return types[0] + # An `Optional` or `Union` type, check to ensure non-heterogenity. if type_origin is Union: python_type = _get_non_heterogenous_type(get_args(type_annotation)) @@ -194,13 +206,13 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType: if python_type is not Decimal: return get_type_from_annotation(python_type) - try: # Grab the decimal configuration from the list of other args. - configuration: DecimalConfig = next( - filter(lambda config: isinstance(config, DecimalConfig), other_args) - ) - except StopIteration: + config_options = [arg for arg in other_args if hasattr(arg, "max_digits")] + if config_options: + configuration = config_options[0] + else: configuration = DEFAULT_DECIMAL_CONFIG - return st.DecimalType(configuration.precision, configuration.scale) + + return st.DecimalType(configuration.max_digits, configuration.decimal_places) # Ensure that we have a concrete type at this point. if not isinstance(type_annotation, type): @@ -234,11 +246,6 @@ def get_type_from_annotation(type_annotation: Any) -> st.DataType: return st.StructType(fields) - if issubclass(type_annotation, ConstrainedDecimal): - precision = int(type_annotation.max_digits or 38) - scale = int(type_annotation.decimal_places or precision) - return st.DecimalType(precision, scale) - if type_annotation is list: raise ValueError( f"list must have type annotation (e.g. `list[str]`), got {type_annotation!r}" diff --git a/src/dve/core_engine/backends/implementations/spark/utilities.py b/src/dve/core_engine/backends/implementations/spark/utilities.py index 5eca158..399d355 100644 --- a/src/dve/core_engine/backends/implementations/spark/utilities.py +++ b/src/dve/core_engine/backends/implementations/spark/utilities.py @@ -110,7 +110,7 @@ class PydanticCompatibleJSONEncoder(JSONEncoder): def default(self, o: Any) -> Any: """Sets the format for given types for json encoding""" if isinstance(o, BaseModel): - return o.dict() + return o.model_dump() if isinstance(o, dt.date): return o.isoformat() return super().default(o) diff --git a/src/dve/core_engine/backends/metadata/contract.py b/src/dve/core_engine/backends/metadata/contract.py index b233e92..e3eb0c0 100644 --- a/src/dve/core_engine/backends/metadata/contract.py +++ b/src/dve/core_engine/backends/metadata/contract.py @@ -2,7 +2,7 @@ from typing import Any -from pydantic import BaseModel, PrivateAttr, root_validator +from pydantic import BaseModel, PrivateAttr, model_validator from dve.core_engine.type_hints import EntityName, ReportingFields from dve.core_engine.validation import RowValidator @@ -47,7 +47,7 @@ def schemas(self) -> dict[EntityName, type[BaseModel]]: self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137 return self._schemas.copy() # pylint: disable=E1101 - @root_validator(allow_reuse=True) + @model_validator(mode="before") @classmethod def _ensure_entities_complete(cls, values: dict[str, dict[EntityName, Any]]): """Ensure the entities in 'readers' and 'validators' are the same.""" diff --git a/src/dve/core_engine/backends/metadata/reporting.py b/src/dve/core_engine/backends/metadata/reporting.py index 3989a13..6385f37 100644 --- a/src/dve/core_engine/backends/metadata/reporting.py +++ b/src/dve/core_engine/backends/metadata/reporting.py @@ -5,7 +5,7 @@ from collections.abc import Callable from typing import Any, ClassVar, Optional, Union -from pydantic import BaseModel, root_validator, validate_arguments +from pydantic import BaseModel, model_validator, validate_call from typing_extensions import Literal from dve.core_engine.templating import template_object @@ -124,8 +124,10 @@ def template( variables.update(local_variables) else: variables = local_variables - templated = template_object(self.dict(exclude=self.UNTEMPLATED_FIELDS), variables, "jinja") - templated.update(self.dict(include=self.UNTEMPLATED_FIELDS)) + templated = template_object( + self.model_dump(exclude=self.UNTEMPLATED_FIELDS), variables, "jinja" + ) + templated.update(self.model_dump(include=self.UNTEMPLATED_FIELDS)) return type_(**templated) @@ -269,7 +271,7 @@ class LegacyReportingConfig(BaseReportingConfig): legacy_is_informational: Optional[Union[bool, str]] = None """DEPRECATED: The legacy 'is_informational' flag.""" - @root_validator(allow_reuse=True, skip_on_failure=True) + @model_validator(mode="before") @classmethod def _ensure_only_one_reporting_config(cls, values: dict[str, Any]) -> dict[str, Any]: """Ensure only the modern or legacy location is populated.""" @@ -283,7 +285,7 @@ def _ensure_only_one_reporting_config(cls, values: dict[str, Any]) -> dict[str, ) return values - @root_validator(allow_reuse=True, skip_on_failure=True) + @model_validator(mode="before") @classmethod def _ensure_only_one_error_type_config(cls, values: dict[str, Any]) -> dict[str, Any]: """Ensure only the modern or legacy error type is populated.""" @@ -300,7 +302,7 @@ def _ensure_only_one_error_type_config(cls, values: dict[str, Any]) -> dict[str, return values @staticmethod - @validate_arguments + @validate_call def _convert_legacy_emit_value( failure_type: Literal["record", "submission", "integrity", "group"], is_informational: bool ) -> str: @@ -319,7 +321,7 @@ def _convert_legacy_emit_value( return emit @staticmethod - @validate_arguments + @validate_call def _convert_legacy_reporting_fields( error_location: Optional[str] = None, reporting_field: Union[str, list[str], None] = None ) -> Optional[str]: diff --git a/src/dve/core_engine/backends/metadata/rules.py b/src/dve/core_engine/backends/metadata/rules.py index 7bc0353..f3a6305 100644 --- a/src/dve/core_engine/backends/metadata/rules.py +++ b/src/dve/core_engine/backends/metadata/rules.py @@ -1,11 +1,13 @@ """Metadata classes for rule steps.""" +from __future__ import annotations + import warnings from abc import ABCMeta, abstractmethod from collections.abc import Iterator, Sequence from typing import Any, ClassVar, Optional, TypeVar, Union -from pydantic import BaseModel, Extra, Field, root_validator, validate_arguments, validator +from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator from typing_extensions import Literal from dve.core_engine.backends.base.reference_data import ReferenceConfigUnion @@ -72,16 +74,15 @@ def __repr__(self) -> str: else: components.append(f"rule=Rule(name={self.rule!r}, ...)") - for key, value in self.dict(exclude={"rule"}).items(): + for key, value in self.model_dump(exclude={"rule"}).items(): components.append(f"{key}={value!r}") return f"ParentMetadata({', '.join(components)})" - class Config: # pylint: disable=too-few-public-methods - """`pydantic configuration options`""" - - frozen = True - extra = Extra.forbid + model_config = { + "frozen": True, + "extra": "allow", + } # pylint: disable=too-few-public-methods @@ -98,11 +99,10 @@ class AbstractStep(BaseModel, metaclass=ABCMeta): UNTEMPLATED_KEYS: ClassVar[set[str]] = {"id", "description", "parent"} """A set of aliases which are exempted from templating.""" - class Config: # pylint: disable=too-few-public-methods - """`pydantic configuration options`""" - - frozen = True - extra = Extra.forbid + model_config = { + "frozen": True, + "extra": "allow", + } def __repr_args__(self) -> Sequence[tuple[Optional[str], Any]]: # Exclude nulls from 'repr' for conciseness. @@ -149,7 +149,7 @@ def template( def __str__(self): # pydantic's default __str__ strips the model name. return super().__repr__() - @root_validator(pre=True) + @model_validator(mode="before") @classmethod def _warn_for_deprecated_aliases(cls, values: dict[str, JSONable]) -> dict[str, JSONable]: for deprecated_name, replacement in ( @@ -375,27 +375,19 @@ class Aggregation(BaseStep): agg_function: Optional[Alias] = None """The aggregate function to apply to the agg_columns (for duckdb backend)""" - @validator("pivot_values") + @field_validator("pivot_values") @classmethod - def _ensure_column_if_values( - cls, - value: Optional[Any], - values: dict[str, Any], - ): + def _ensure_column_if_values(cls, value: Optional[Any], info: ValidationInfo): """Ensure that `pivot_column` is not null if pivot values are provided.""" - if value and not values["pivot_column"]: + if value and not info.data["pivot_column"]: raise ValueError("`pivot_values` specified, but no `pivot_column`") return value - @validator("agg_function") + @field_validator("agg_function") @classmethod - def _ensure_column_if_function( - cls, - agg_function: Optional[Any], - values: dict[str, Any], - ): + def _ensure_column_if_function(cls, agg_function: Optional[Any], info: ValidationInfo): """Ensure that `pivot_column` is not null if pivot values are provided.""" - if agg_function and not values["agg_columns"]: + if agg_function and not info.data["agg_columns"]: raise ValueError("`agg_function` specified, but no `agg_columns`") return agg_function @@ -582,7 +574,7 @@ def __str__(self): # pydantic's default __str__ strips the model name. return super().__repr__() @classmethod - @validate_arguments + # @validate_call #TODO - removed for now as it's broken in pydantic v2 def from_step_list(cls, name: str, steps: list[Step]): """Load the rule from a single step list.""" pre_sync_steps: list[AbstractStep] = [] @@ -711,7 +703,7 @@ class RuleMetadata(BaseModel): """ - @root_validator() + @model_validator(mode="before") @classmethod def _ensure_locals_same_length_as_rules(cls, values: dict[str, list[Any]]): """Ensure that if 'local_variables' is provided, it's the same length as 'rules'.""" @@ -734,4 +726,4 @@ def __iter__(self) -> Iterator[tuple[Rule, TemplateVariables]]: # type: ignore yield from zip(self.rules, self.local_variables) -ParentMetadata.update_forward_refs() +ParentMetadata.model_rebuild() diff --git a/src/dve/core_engine/backends/readers/csv.py b/src/dve/core_engine/backends/readers/csv.py index dbace5b..cfa2dcd 100644 --- a/src/dve/core_engine/backends/readers/csv.py +++ b/src/dve/core_engine/backends/readers/csv.py @@ -238,7 +238,7 @@ def read_to_py_iterator( if self.field_check: self.perform_field_check(resource, entity_name, schema, all_model_fields) - field_names = list(schema.__fields__.keys()) + field_names = list(schema.model_fields.keys()) with open_stream(resource, "r", self.encoding) as stream: reader = csv.DictReader( stream, @@ -265,8 +265,8 @@ def write_parquet( # type: ignore target_location = file_uri_to_local_path(target_location).as_posix() if schema: polars_schema: dict[str, pl.DataType] = { # type: ignore - fld.name: get_polars_type_from_annotation(fld.annotation) - for fld in stringify_model(schema).__fields__.values() + name: get_polars_type_from_annotation(fld.annotation) + for name, fld in stringify_model(schema).model_fields.items() } polars_schema[RECORD_INDEX_COLUMN_NAME] = get_polars_type_from_annotation(int) diff --git a/src/dve/core_engine/backends/readers/utilities.py b/src/dve/core_engine/backends/readers/utilities.py index 00086bd..3948d70 100644 --- a/src/dve/core_engine/backends/readers/utilities.py +++ b/src/dve/core_engine/backends/readers/utilities.py @@ -21,7 +21,8 @@ def check_csv_header_expected( """Check the header of a CSV matches the expected fields""" with open_stream(resource) as fle: header_fields = fle.readline().rstrip().replace(quote_char, "").split(delimiter) - expected_fields = expected_schema.__fields__.keys() + + expected_fields = expected_schema.model_fields.keys() missing = set(expected_fields).difference(header_fields) additional = set(header_fields).difference(all_model_fields) @@ -72,4 +73,4 @@ def raise_message_bearing_error_on_header_differences( def get_all_model_fields(models: Iterable[type[BaseModel]]) -> set[str]: """Return all field names from all available models""" - return {field for model in models for field in model.__fields__.keys()} + return {field for model in models for field in model.model_fields.keys()} diff --git a/src/dve/core_engine/backends/readers/xml.py b/src/dve/core_engine/backends/readers/xml.py index 13a7c38..badc03a 100644 --- a/src/dve/core_engine/backends/readers/xml.py +++ b/src/dve/core_engine/backends/readers/xml.py @@ -13,7 +13,12 @@ from dve.core_engine.backends.base.reader import BaseFileReader from dve.core_engine.backends.exceptions import EmptyFileError from dve.core_engine.backends.readers.xml_linting import run_xmllint -from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model +from dve.core_engine.backends.utilities import ( + get_polars_type_from_annotation, + is_field_complex, + is_type_complex, + stringify_model, +) from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME from dve.core_engine.loggers import get_logger from dve.core_engine.message import FeedbackMessage @@ -43,6 +48,15 @@ def _strip_annotated(annotation: Any) -> Any: return get_args(annotation)[0] +def _strip_optional(annotation: Any) -> Any: + """Strip Optional type from a type""" + if hasattr(annotation, "_name"): + if annotation._name == "Optional": # pylint: disable=W0212 + python_type, _default = get_args(annotation) + return python_type + return annotation + + def create_template_row(schema: type[BaseModel]) -> dict[str, Any]: """Create a template row from a schema. A template row is essentially the shape of the record that would be populated by the reader (i.e. contains @@ -51,10 +65,10 @@ def create_template_row(schema: type[BaseModel]) -> dict[str, Any]: """ template_row: dict[str, Any] = {} - for field_name, model_field_def in schema.__fields__.items(): - field_type = _strip_annotated(model_field_def.annotation) + for field_name, model_field_def in schema.model_fields.items(): # type: ignore + field_type = _strip_optional(_strip_annotated(model_field_def.annotation)) - if not model_field_def.is_complex(): + if not is_type_complex(field_type): template_row[field_name] = None continue @@ -70,8 +84,8 @@ def create_template_row(schema: type[BaseModel]) -> dict[str, Any]: # This is a quick and dirty hack to avoid implementing our own logic # to check complex types... - list_type_field_spec = create_model("", lt=(list_type, ...)).__fields__["lt"] - if not list_type_field_spec.is_complex(): + list_type_field_spec = create_model("", lt=(list_type, ...)).model_fields["lt"] + if not is_field_complex(list_type_field_spec): template_row[field_name] = [None] continue @@ -330,8 +344,8 @@ def write_parquet( # type: ignore target_location = file_uri_to_local_path(target_location).as_posix() if schema: polars_schema: dict[str, pl.DataType] = { # type: ignore - fld.name: get_polars_type_from_annotation(fld.type_) - for fld in stringify_model(schema).__fields__.values() + name: get_polars_type_from_annotation(fld.type_) + for name, fld in stringify_model(schema).model_fields.items() } polars_schema[RECORD_INDEX_COLUMN_NAME] = get_polars_type_from_annotation(int) pl.LazyFrame(data=entity, schema=polars_schema).sink_parquet( diff --git a/src/dve/core_engine/backends/utilities.py b/src/dve/core_engine/backends/utilities.py index 66f4750..af2f757 100644 --- a/src/dve/core_engine/backends/utilities.py +++ b/src/dve/core_engine/backends/utilities.py @@ -1,14 +1,16 @@ """Necessary, otherwise uncategorised backend functionality.""" +import inspect import sys from dataclasses import is_dataclass from datetime import date, datetime, time from decimal import Decimal -from typing import Any, ClassVar, GenericAlias, Union # type: ignore +from typing import Any, ClassVar, GenericAlias, Literal, Union # type: ignore import polars as pl # type: ignore from polars.datatypes.classes import DataTypeClass as PolarsType from pydantic import BaseModel, create_model +from pydantic.fields import FieldInfo from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME @@ -81,6 +83,27 @@ def datetime_format_to_regex(format_str: str) -> str: """A mapping of Python types to the equivalent Polars types.""" +def is_type_complex(type_: Any) -> bool: + """Check whether a type is a complex type or not.""" + _simple_types = (str, int, float, bool, bytes, date, datetime, time, Decimal) + + if type_ in _simple_types or ( + inspect.isclass(type_) and any(issubclass(type_, st) for st in _simple_types) + ): + return False + + return True + + +def is_field_complex(field: FieldInfo) -> bool: + """ + Replacement function for Pydantic v1 `is_complex` check provided + by the v1 ModelField object. + """ + type_annotation = field.annotation + return is_type_complex(type_annotation) + + def stringify_type(type_: Union[type, GenericAlias]) -> type: """Stringify an individual type.""" if isinstance(type_, type) and not isinstance( @@ -89,7 +112,7 @@ def stringify_type(type_: Union[type, GenericAlias]) -> type: if issubclass(type_, BaseModel): return stringify_model(type_) - is_complex = create_model("", t=(type_, ...)).__fields__["t"].is_complex() + is_complex = is_field_complex(create_model("", t=(type_, ...)).model_fields["t"]) if not is_complex: # A non-container type, return string. return str @@ -111,9 +134,9 @@ def stringify_type(type_: Union[type, GenericAlias]) -> type: def stringify_model(model: type[BaseModel]) -> type[BaseModel]: """Stringify a `pydantic` model.""" fields = {} - for field_name, field in model.__fields__.items(): - fields[field_name] = (stringify_type(field.annotation), ...) - return create_model(model.__name__, **fields) # type: ignore + for field_name, field in model.model_fields.items(): + fields[field_name] = (stringify_type(field.annotation), ...) # type: ignore + return create_model(model.__class__.__name__, **fields) # type: ignore def dedup_messages(messages: Messages) -> Messages: @@ -149,7 +172,7 @@ def get_polars_type_from_annotation(type_annotation: Any) -> PolarsType: 'optional' wrapper and return the inner type - A subclass of `typing.TypedDict` with values typed using supported types. This will parse the value types as Polars types and return a Polars Struct. - - A dataclass or `pydantic.main.ModelMetaClass` with values typed using supported types. + - A dataclass or `pydantic.BaseModel` with values typed using supported types. This will parse the field types as Polars types and return a Polars Struct. - Any supported type, with a `typing_extensions.Annotated` wrapper. - A `decimal.Decimal` wrapped with `typing_extensions.Annotated` with a `DecimalConfig` @@ -163,6 +186,15 @@ def get_polars_type_from_annotation(type_annotation: Any) -> PolarsType: """ type_origin = get_origin(type_annotation) + if type_origin is Literal: + # TODO - look at using _get_non_heterogenous_type instead? + polars_types = [get_polars_type_from_annotation(type(t)) for t in get_args(type_annotation)] + if not polars_types or not all(t == polars_types[0] for t in polars_types): + raise ValueError( + f"Unable to determine a single concrete type for Literal. Got {type_annotation!r}" + ) + return polars_types[0] + # An `Optional` or `Union` type, check to ensure non-heterogenity. if type_origin is Union: python_type = _get_non_heterogenous_type(get_args(type_annotation)) diff --git a/src/dve/core_engine/configuration/v1/__init__.py b/src/dve/core_engine/configuration/v1/__init__.py index 6221957..959596f 100644 --- a/src/dve/core_engine/configuration/v1/__init__.py +++ b/src/dve/core_engine/configuration/v1/__init__.py @@ -3,8 +3,8 @@ import json from typing import Any, Optional, Union -from pydantic import BaseModel, Field, PrivateAttr, validate_arguments -from typing_extensions import Annotated, Literal +from pydantic import BaseModel, Field, PrivateAttr, validate_call +from typing_extensions import Literal from dve.core_engine.backends.base.reference_data import ReferenceConfig, ReferenceConfigUnion from dve.core_engine.backends.metadata.contract import DataContractMetadata, ReaderConfig @@ -178,7 +178,7 @@ class V1EngineConfig(BaseEngineConfig): ) """Rule store rules from the loaded rule stores.""" - @validate_arguments + @validate_call def _update_rule_store(self, rule_store: dict[RuleName, BusinessComponentSpecConfigUnion]): """Update the rule store rules to add/override the rules from the new store.""" self._rule_store_rules.update(rule_store) # pylint: disable=E1101 @@ -282,8 +282,8 @@ def _load_rules_and_vars(self) -> tuple[list[Rule], list[TemplateVariables]]: added_rules: set[RuleName] = set() for index, complex_rule_config in enumerate( - self.transformations.complex_rules - ): # pylint: disable=E1101 + self.transformations.complex_rules # pylint: disable=E1101 + ): rule, local_params, deps = self._resolve_business_rule(complex_rule_config) missing_rules = deps - added_rules if missing_rules: @@ -311,7 +311,7 @@ def get_contract_metadata(self) -> DataContractMetadata: validators = {} reporting_fields = {} - contract_dict = self.contract.dict() + contract_dict = self.contract.model_dump() error_info = {} if self.contract.error_details: error_info = self.load_error_message_info(self.contract.error_details) diff --git a/src/dve/core_engine/configuration/v1/steps.py b/src/dve/core_engine/configuration/v1/steps.py index f795c8c..6b30a4a 100644 --- a/src/dve/core_engine/configuration/v1/steps.py +++ b/src/dve/core_engine/configuration/v1/steps.py @@ -10,9 +10,9 @@ # pylint: disable=missing-class-docstring from abc import ABC, abstractmethod -from typing import Any, Optional, Union +from typing import Optional, Union -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, Field, ValidationInfo, field_validator from typing_extensions import Annotated, Literal from dve.core_engine.backends.metadata.rules import ( @@ -40,10 +40,9 @@ class ConfigStep(BaseModel, ABC): """The parent for the config steps.""" - class Config: # pylint: disable=too-few-public-methods - """Config class for dynamically generated pydantic models""" - - extra = Extra.forbid + model_config = { + "extra": "forbid", + } name: Optional[str] = None """The 'name' of the rule. This is mapped to an ID in the entity.""" @@ -107,10 +106,10 @@ class GroupByConfig(ConfigStep): pivot_values: Optional[list[str]] = None agg_columns: MultipleExpressions - @validator("pivot_values") + @field_validator("pivot_values") @classmethod - def _ensure_no_values_if_not_column(cls, value: Optional[str], values: dict[str, Any]): - if value and not values["pivot_column"]: + def _ensure_no_values_if_not_column(cls, value: Optional[str], info: ValidationInfo): + if value and not info.data.get("pivot_column"): raise ValueError("Cannot provide 'pivot_values' if no 'pivot_column'") return value diff --git a/src/dve/core_engine/engine.py b/src/dve/core_engine/engine.py index 838bfb5..a7931ad 100644 --- a/src/dve/core_engine/engine.py +++ b/src/dve/core_engine/engine.py @@ -4,9 +4,17 @@ import logging from pathlib import Path from types import TracebackType -from typing import Any, Optional, Union - -from pydantic import BaseModel, Field, PrivateAttr, validate_arguments, validator +from typing import Annotated, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PrivateAttr, + ValidationInfo, + field_validator, + validate_call, +) from pydantic.types import FilePath from pyspark.sql import SparkSession @@ -26,11 +34,7 @@ class CoreEngine(BaseModel): """The core engine implementation for the data validation engine.""" - class Config: # pylint: disable=too-few-public-methods - """`pydantic` configuration options.""" - - arbitrary_types_allowed = True - validate_assignment = True + model_config = {"arbitrary_types_allowed": True, "validate_assignment": True} backend_config: BaseEngineConfig """The backend configuration for the given run.""" @@ -52,16 +56,20 @@ class Config: # pylint: disable=too-few-public-methods Data will be chunked to parquet in this directory after being read, and written here before filters are applied. - """ - backend: BaseBackend = None # type: ignore + # TODO - recommended to not use validate_default like this, but for now will use as replacement to always=True # pylint: disable=C0301 + # see https://pydantic.dev/docs/validation/latest/get-started/migration/#validator-and-root_validator-are-deprecated # pylint: disable=C0301 + backend: Annotated[Optional[BaseBackend], Field(default=None, validate_default=True)] """The backend to use to process the files.""" + debug: bool = False """Indication of if this run is in debug mode.""" - @validator("cache_prefix_uri", "output_prefix_uri", allow_reuse=True, pre=True) + @field_validator("cache_prefix_uri", "output_prefix_uri", mode="before") # pylint: disable=E0213 - def _validate_prefix_uri(cls, location: Optional[Location]) -> Optional[URI]: + def _validate_prefix_uri( + cls, location: Optional[Location], info: ValidationInfo # pylint: disable=W0613 + ) -> Optional[URI]: """Ensure we support the cache prefix scheme.""" if location is None: return None @@ -71,25 +79,25 @@ def __init__(self, *args, **kwargs): # pylint: disable=W0235 super().__init__(*args, **kwargs) - @validator("backend", always=True) + @field_validator("backend") @classmethod - def _ensure_backend(cls, value: Optional[BaseBackend], values: dict[str, Any]) -> BaseBackend: + def _ensure_backend(cls, value: Optional[BaseBackend], info: ValidationInfo) -> BaseBackend: """Ensure a default backend is created if a backend is not specified.""" if value is not None: return value - main_logger = values.get("main_log") + main_logger = info.data.get("main_log") if main_logger is None: - return SparkBackend(dataset_config_uri=values.get("dataset_config_uri")) + return SparkBackend(dataset_config_uri=info.data.get("dataset_config_uri")) return SparkBackend( - dataset_config_uri=values.get("dataset_config_uri"), + dataset_config_uri=info.data.get("dataset_config_uri"), logger=get_child_logger( ".".join((SparkBackend.__module__, SparkBackend.__name__)), main_logger ), ) @classmethod - @validate_arguments(config={"arbitrary_types_allowed": True}) + @validate_call(config=ConfigDict(arbitrary_types_allowed=True)) def build( cls, dataset_config_path: Union[FilePath, URI], @@ -152,7 +160,7 @@ def build_from_model(cls, model_str: JSONstring): """ main_log = get_logger("CoreEngine") main_log.info("Initalise from model...") - return cls.build(**EngineRunValidation(**json.loads(model_str)).dict()) + return cls.build(**EngineRunValidation(**json.loads(model_str)).model_dump()) def __enter__(self) -> "CoreEngine": self.main_log.info("Entering pipeline context.") # pylint: disable=E1101 @@ -170,9 +178,9 @@ def __exit__( exc_value: Optional[Exception], traceback: Optional[TracebackType], ) -> None: - self.main_log.info( + self.main_log.info( # pylint: disable=E1101 f"Exiting pipeline context, clearing {self.cache_prefix!r}" - ) # pylint: disable=E1101 + ) cache_dir = self._cache_dir self._cache_dir = None @@ -200,9 +208,9 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities: """ output_entities = {} - self.main_log.info( + self.main_log.info( # pylint: disable=E1101 f"Writing entities to the output location: {self.output_prefix_uri}" - ) # pylint: disable=E1101 + ) for entity_name, entity in entities.items(): entity = entity.drop(RECORD_INDEX_COLUMN_NAME) @@ -210,13 +218,13 @@ def _write_entity_outputs(self, entities: SparkEntities) -> SparkEntities: output_uri = joinuri(self.output_prefix_uri, entity_name) if get_resource_exists(output_uri): - self.main_log.info( + self.main_log.info( # pylint: disable=E1101 f"{output_uri} already exists - will be overwritten" - ) # pylint: disable=E1101 + ) - self.main_log.info( + self.main_log.info( # pylint: disable=E1101 f"+ Writing parquet output to {output_uri!r}" - ) # pylint: disable=E1101 + ) entity.write.mode("overwrite").parquet(output_uri) spark_session = SparkSession.builder.getOrCreate() output_entities[entity_name] = spark_session.read.format("parquet").load( @@ -263,11 +271,16 @@ def run_pipeline( references should be valid after the pipeline context exits. """ - entities, errors_uri = self.backend.process_legacy( - self.output_prefix_uri, - entity_locations, - self.backend_config.get_contract_metadata(), - self.backend_config.get_rule_metadata(), - submission_info, + if self.backend: + entities, errors_uri = self.backend.process_legacy( + self.output_prefix_uri, + entity_locations, + self.backend_config.get_contract_metadata(), + self.backend_config.get_rule_metadata(), + submission_info, + ) + return self._write_outputs(entities), errors_uri + + raise AttributeError( + "Backend implementation not defined. Cannot run the pipeline without a defined backend. Choose DuckDB or Spark" # pylint: disable=C0301 ) - return self._write_outputs(entities), errors_uri diff --git a/src/dve/core_engine/loggers.py b/src/dve/core_engine/loggers.py index 036c3d2..6c9b3aa 100644 --- a/src/dve/core_engine/loggers.py +++ b/src/dve/core_engine/loggers.py @@ -18,7 +18,7 @@ def filter(self, record): class UTCFormatter(logging.Formatter): # pragma: no cover """A formatter with timestamps in the UTC timezone.""" - converter = time.gmtime + converter = time.gmtime # type: ignore def get_default_handler() -> logging.Handler: # pragma: no cover diff --git a/src/dve/core_engine/message.py b/src/dve/core_engine/message.py index 626a710..78024e9 100644 --- a/src/dve/core_engine/message.py +++ b/src/dve/core_engine/message.py @@ -10,7 +10,7 @@ from functools import reduce from typing import Any, ClassVar, Optional, Union -from pydantic import BaseModel, ValidationError, validator +from pydantic import BaseModel, ConfigDict, ValidationError, ValidationInfo, field_validator from pydantic.dataclasses import dataclass from dve.core_engine.constants import CONTRACT_ERROR_VALUE_FIELD_NAME, RECORD_INDEX_COLUMN_NAME @@ -85,12 +85,6 @@ def extract_error_value(records, error_location): """ -class Config: # pylint: disable=too-few-public-methods - """`pydantic` configuration options.""" - - arbitrary_types_allowed = True - - # pylint: disable=R0902 @dataclass class UserMessage: @@ -132,7 +126,7 @@ def is_critical(self) -> bool: return self.FailureType == "integrity" -@dataclass(config=Config, eq=True) +@dataclass(config=ConfigDict(arbitrary_types_allowed=True), eq=True) class FeedbackMessage: # pylint: disable=too-many-instance-attributes """Information which affects processing and needs to be feeded back.""" @@ -197,9 +191,11 @@ class FeedbackMessage: # pylint: disable=too-many-instance-attributes ] """The header that should be written to CSV.""" - @validator("reporting_field") + @field_validator("reporting_field") # pylint: disable=no-self-argument - def _split_reporting_field(cls, value) -> Union[list[str], str, None]: + def _split_reporting_field( + cls, value: Optional[str | list[str]], info: ValidationInfo # pylint: disable=W0613 + ) -> Union[list[str], str, None]: if isinstance(value, list): return value if isinstance(value, str): @@ -212,9 +208,11 @@ def _split_reporting_field(cls, value) -> Union[list[str], str, None]: return value return None - @validator("error_location", pre=True) + @field_validator("error_location", mode="before") # pylint: disable=no-self-argument - def _validate_error_location(cls, value: Any) -> Optional[str]: + def _validate_error_location( + cls, value: Optional[str], info: ValidationInfo # pylint: disable=W0613 + ) -> Optional[str]: """Format error location to a string.""" if value is None: return None # pragma: no cover @@ -249,8 +247,8 @@ def from_pydantic_error( messages: Messages = [] for error_dict in error.errors(): error_type = error_dict["type"] - # TODO - review in pydantic v2 - how handles null vs not provided values - if "none.not_allowed" in error_type or "value_error.missing" in error_type: + _input = error_dict["input"] + if "missing" in error_type or _input is None: category = "Blank" else: category = "Bad value" diff --git a/src/dve/core_engine/models.py b/src/dve/core_engine/models.py index f29889a..bba2986 100644 --- a/src/dve/core_engine/models.py +++ b/src/dve/core_engine/models.py @@ -11,7 +11,15 @@ from pathlib import Path from typing import Any, Optional -from pydantic import UUID4, BaseModel, Field, FilePath, root_validator, validator +from pydantic import ( + UUID4, + BaseModel, + Field, + FilePath, + ValidationInfo, + field_validator, + model_validator, +) from dve.core_engine.backends.metadata.contract import ReaderConfig from dve.core_engine.type_hints import EntityName, ProcessingStatus, SubmissionResult @@ -26,16 +34,17 @@ class AuditRecord(BaseModel): submission_id: str """Unique id of the submission""" + # todo - why bother supplying a date_updated here when it gets overwritten by time? Only works if you supply both values # pylint: disable=C0301 date_updated: Optional[dt.date] = None """The date the record was added to the table""" time_updated: Optional[dt.datetime] = Field(default_factory=dt.datetime.now) """The timestamp the record was added to the table""" - @root_validator(allow_reuse=True) - def populate_date_updated(cls, values): # pylint: disable=no-self-argument + @model_validator(mode="after") + def populate_date_updated(self): """Add date_updated from time_updated value""" - values["date_updated"] = values["time_updated"].date() - return values + self.date_updated = self.time_updated.date() # pylint: disable=E1101 + return self class SubmissionInfoMismatchWarning(UserWarning): @@ -64,8 +73,10 @@ class SubmissionInfo(AuditRecord): datetime_received: Optional[dt.datetime] = None # type: ignore """The datetime the file was received.""" - @validator("file_extension") - def _ensure_just_file_stem(cls, extension: str): # pylint: disable=no-self-argument + @field_validator("file_extension") + def _ensure_just_file_stem( + cls, extension: str, info: ValidationInfo # pylint: disable=W0613 + ): # pylint: disable=no-self-argument return extension.rsplit(".", 1)[-1] @property @@ -95,8 +106,8 @@ def __eq__(self, other: object) -> bool: if not isinstance(other, SubmissionInfo): raise NotImplementedError("Unable to determine equality if not a SubmissionInfo object") _exclude = ["date_updated", "time_updated"] - return {k: v for k, v in self.dict().items() if k not in _exclude} == { - k: v for k, v in other.dict().items() if k not in _exclude + return {k: v for k, v in self.model_dump().items() if k not in _exclude} == { + k: v for k, v in other.model_dump().items() if k not in _exclude } @@ -118,8 +129,8 @@ def __eq__(self, other: object) -> bool: "Unable to determine equality if not a SubmissionStatisticsRecord object" ) # pylint: disable=line-too-long _exclude = ["date_updated", "time_updated"] - return {k: v for k, v in self.dict().items() if k not in _exclude} == { - k: v for k, v in other.dict().items() if k not in _exclude + return {k: v for k, v in self.model_dump().items() if k not in _exclude} == { + k: v for k, v in other.model_dump().items() if k not in _exclude } @@ -141,9 +152,9 @@ class ProcessingStatusRecord(AuditRecord): processing_status: ProcessingStatus """The processing status of the submission""" - job_run_id: Optional[int] + job_run_id: Optional[int] = None """The run id of the databricks job used to process the submission""" - submission_result: Optional[SubmissionResult] + submission_result: Optional[SubmissionResult] = None """Whether the file validation was a success or failure""" @@ -158,9 +169,9 @@ class EngineRun(BaseModel): # TODO: What if we want to set an alt/override output prefix # and not have the submission_id appended to it? - @validator("output_prefix") - def _set_output_path(cls, prefix, values: dict): # pylint: disable=E0213 - v_id = values.get("submission_id") + @field_validator("output_prefix") + def _set_output_path(cls, prefix, info: ValidationInfo): # pylint: disable=E0213 + v_id = info.data.get("submission_id") if v_id: return os.path.join(prefix, str(v_id)) return prefix @@ -183,8 +194,11 @@ class ConcreteEntity(EntitySpecification, arbitrary_types_allowed=True): """An optional key field to use for the entity.""" reporting_fields: Optional[list[str]] = None - @validator("reporting_fields", pre=True) - def _ensure_list(cls, value: Optional[str]) -> Optional[list[str]]: # pylint: disable=E0213 + @field_validator("reporting_fields", mode="before") + @classmethod + def _ensure_list( + cls, value: Optional[str], info: ValidationInfo # pylint: disable=W0613 + ) -> Optional[list[str]]: """Ensure the reporting fields are a list.""" if value is None: return None diff --git a/src/dve/core_engine/validation.py b/src/dve/core_engine/validation.py index f62309b..05f2e2b 100644 --- a/src/dve/core_engine/validation.py +++ b/src/dve/core_engine/validation.py @@ -6,8 +6,7 @@ from typing import Optional from pyarrow.lib import RecordBatch # type: ignore -from pydantic import ValidationError -from pydantic.main import ModelMetaclass +from pydantic import BaseModel, ValidationError from dve.core_engine.message import DEFAULT_ERROR_DETAIL, DataContractErrorDetail, FeedbackMessage from dve.core_engine.type_hints import ContractContents, EntityName, ErrorCategory, Messages, Record @@ -36,7 +35,7 @@ def __init__( self._model_definition = model_definition self._validators = validators self.entity_name = entity_name - self._model: Optional[ModelMetaclass] = None + self._model: Optional[BaseModel] = None self._error_info = error_info or {} self._error_details: Optional[ dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]] @@ -48,7 +47,7 @@ def __reduce__(self): # Don't attempt to pickle Pydantic models. return super().__reduce__() @property - def model(self) -> ModelMetaclass: + def model(self) -> BaseModel: """The loaded pydantic model for the entity.""" if not self._model: models = JSONtoPyd(self._model_definition).generate_models( @@ -83,7 +82,7 @@ def __call__(self, record: Record) -> tuple[Optional[Record], Messages]: messages: Messages = [] try: # pylint: disable=not-callable - validated: Record = self.model(**record).dict() + validated: Record = self.model(**record).model_dump() # type: ignore except ValidationError as err: # we still want to report warnings # when a record is invalid diff --git a/src/dve/metadata_parser/domain_types.py b/src/dve/metadata_parser/domain_types.py index 84dc5a7..3d7bc3c 100644 --- a/src/dve/metadata_parser/domain_types.py +++ b/src/dve/metadata_parser/domain_types.py @@ -1,15 +1,18 @@ """Domain specific type definitions for use in validators.""" # pylint: disable=too-few-public-methods +# pylint: disable=W0613 + import datetime as dt import itertools import re import warnings -from collections.abc import Iterator, Sequence +from collections.abc import Sequence from functools import lru_cache -from typing import ClassVar, Optional, TypeVar, Union +from typing import Any, ClassVar, Optional, TypeVar, Union -from pydantic import fields, types, validate_arguments +from pydantic import GetCoreSchemaHandler, types, validate_call +from pydantic_core import CoreSchema, core_schema from typing_extensions import Literal from dve.metadata_parser import exc @@ -32,16 +35,16 @@ POSTCODE_REGEX = re.compile(r"\A[a-zA-Z]{1,2}\d([a-zA-Z]?|\d?)\s\d[a-zA-Z]{2}\Z") -class _SimpleRegexValidator(types.ConstrainedStr): +class _SimpleRegexValidator: """A basic regex-validated type.""" - regex: re.Pattern + pattern: re.Pattern """A regex pattern used to validate the string.""" strip_whitespace: bool = True """Whether to strip the whitespace from the string.""" -class NHSNumber(types.ConstrainedStr): +class NHSNumber(str): """A constrained string which validates an NHS number. ### Validation criteria @@ -98,7 +101,7 @@ class NHSNumber(types.ConstrainedStr): warn_on_test_numbers = True @classmethod - def _warn_for_possible_invalid_number(cls, nhs_number: str, loc: str) -> None: + def _warn_for_possible_invalid_number(cls, nhs_number: str) -> None: """Emit warnings for possible invalid NHS numbers.""" reason = None @@ -111,10 +114,10 @@ def _warn_for_possible_invalid_number(cls, nhs_number: str, loc: str) -> None: reason = "NHS number is a palindrome: this indicates a test number" if reason: - warnings.warn(exc.LocWarning(f"NHS number possibly invalid ({reason})", loc)) + warnings.warn(exc.LocWarning(f"NHS number possibly invalid ({reason})")) @staticmethod - def ensure_format(nhs_number: Optional[str]) -> str: + def ensure_format(nhs_number: Optional[str | int]) -> str: """Coerce an NHS number string to the correct format, raising an error if coersion fails. @@ -139,7 +142,7 @@ def confirm_checksum_validates(nhs_number: str) -> bool: return check == int(check_digit) @classmethod - def check_validates(cls, value: Optional[str]) -> bool: + def check_validates(cls, value: str) -> bool: """Check whether an NHS number is valid, returning `True` for valid numbers and `False` for invalid numbers. @@ -152,19 +155,24 @@ def check_validates(cls, value: Optional[str]) -> bool: return is_valid @classmethod - def validate(cls, value: Optional[str], field: fields.ModelField) -> str: # type: ignore # pylint: disable=W0221 + def validate(cls, value: Optional[str | int]) -> str: # type: ignore # pylint: disable=W0221 """Validates the given postcode""" nhs_number = cls.ensure_format(value) if cls.confirm_checksum_validates(nhs_number): - # TODO: Get a better way to get 'loc' here. - cls._warn_for_possible_invalid_number(nhs_number, field.name) + cls._warn_for_possible_invalid_number(nhs_number) return nhs_number raise ValueError("NHS number invalid (incorrect check digit: cannot be a real NHS number)") + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) + @lru_cache() -@validate_arguments +@validate_call def permissive_nhs_number(warn_on_test_numbers: bool = False): """Defaults to not checking for test numbers""" dict_ = NHSNumber.__dict__.copy() @@ -173,10 +181,10 @@ def permissive_nhs_number(warn_on_test_numbers: bool = False): return type("NHSNumber", (NHSNumber, *NHSNumber.__bases__), dict_) -class Postcode(types.ConstrainedStr): +class Postcode(str): """Postcode constrained string""" - regex: re.Pattern = POSTCODE_REGEX + pattern: re.Pattern = POSTCODE_REGEX strip_whitespace = True apply_normalize = True @@ -198,14 +206,25 @@ def validate(cls, value: str) -> Optional[str]: # type: ignore if not value: return None - if not cls.regex.match(value): + if not cls.pattern.match(value): raise ValueError("Invalid Postcode submitted") return value + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.chain_schema( + [ + handler(str), + core_schema.no_info_plain_validator_function(cls.validate), + ] + ) + @lru_cache() -@validate_arguments +@validate_call def postcode( # pylint: disable=R0913 strip_whitespace: Optional[bool] = True, @@ -214,8 +233,7 @@ def postcode( strict: Optional[bool] = False, min_length: Optional[int] = None, max_length: Optional[int] = None, - curtail_length: Optional[int] = None, - regex: Optional[str] = POSTCODE_REGEX, # type: ignore + pattern: Optional[str] = POSTCODE_REGEX, # type: ignore apply_normalize: Optional[bool] = True, ) -> type[Postcode]: """Return a formatted date class with a set date format @@ -229,8 +247,7 @@ def postcode( dict_["strict"] = strict dict_["min_length"] = min_length dict_["max_length"] = max_length - dict_["curtail_length"] = curtail_length - dict_["regex"] = regex + dict_["pattern"] = pattern dict_["apply_normalize"] = apply_normalize return type("Postcode", (Postcode, *Postcode.__bases__), dict_) @@ -244,17 +261,17 @@ class OrgID(_SimpleRegexValidator): """ - regex = re.compile(r"^[A-Z0-9]{3,5}$") + pattern = re.compile(r"^[A-Z0-9]{3,5}$") strip_whitespace = False @classmethod - def validate(cls, value: str) -> str: - """Validates the given OrgID""" - if not value: - raise ValueError("org_id not provided") - return super().validate(value) + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.str_schema(pattern=cls.pattern, strip_whitespace=cls.strip_whitespace) +# TODO - look into replacing datetime types with AwareDatetime and NaiveDatetime from pydantic v2 class ConFormattedDate(dt.date): """A date, provided as a date or a string in a specific format.""" @@ -274,12 +291,9 @@ class ConFormattedDate(dt.date): @classmethod def validate(cls, value: Optional[Union[dt.date, str]]) -> Optional[dt.date]: """Validate a passed datetime or string.""" - if value is None: - return value - if isinstance(value, dt.date): date = value - elif cls.DATE_FORMAT is not None: + elif cls.DATE_FORMAT is not None and value: try: date = dt.datetime.strptime(value, cls.DATE_FORMAT).date() if cls.strict and (date.strftime(cls.DATE_FORMAT) != value): @@ -311,14 +325,20 @@ def validate_range(cls, value) -> Optional[dt.date]: return value @classmethod - def __get_validators__(cls) -> Iterator[classmethod]: - """Gets all validators""" - yield cls.validate # type: ignore - yield cls.validate_range # type: ignore + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.chain_schema( + [ + handler(str), + core_schema.no_info_plain_validator_function(cls.validate), + core_schema.no_info_plain_validator_function(cls.validate_range), + ] + ) @lru_cache() -@validate_arguments +@validate_call def conformatteddate( date_format: Optional[str] = None, strict: Optional[bool] = False, @@ -381,6 +401,7 @@ def reformat_nhs_string_format(string: str) -> str: ) ) + # TODO - check this hasn't broken as pydantic.parse_datetime retired although this not using pydantic datetime AFAIK pylint: disable=C0301 @classmethod def parse_datetime(cls, string: str) -> dt.datetime: """Attempt to parse a datetime using various formats in sequence.""" @@ -403,6 +424,8 @@ def parse_datetime(cls, string: str) -> dt.datetime: @classmethod def validate(cls, value: Optional[Union[dt.datetime, str]]) -> Optional[dt.datetime]: """Validate a passed datetime or string.""" + # TODO - This check is simply needed because of the test in test_domain_types which is possibly invalid post pylint: disable=C0301 + # Pydantic v2 upgrade now as NoneType will be handled by the handler if value is None: return value @@ -427,9 +450,17 @@ def validate(cls, value: Optional[Union[dt.datetime, str]]) -> Optional[dt.datet return datetime @classmethod - def __get_validators__(cls) -> Iterator[classmethod]: + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: """Gets all validators""" - yield cls.validate # type: ignore + # yield cls.validate # type: ignore + return core_schema.chain_schema( + [ + handler(str), + core_schema.no_info_plain_validator_function(cls.validate), + ] + ) class FormattedTime(dt.time): @@ -496,9 +527,6 @@ def parse_time(cls, string: str) -> dt.time: @classmethod def validate(cls, value: Union[dt.time, dt.datetime, str]) -> dt.time | None: """Validate a passed time, datetime or string.""" - if value is None: - return value - if isinstance(value, dt.time): new_time = value elif isinstance(value, dt.datetime): @@ -524,13 +552,20 @@ def validate(cls, value: Union[dt.time, dt.datetime, str]) -> dt.time | None: return new_time @classmethod - def __get_validators__(cls) -> Iterator[classmethod]: + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: """Gets all validators""" - yield cls.validate # type: ignore + return core_schema.chain_schema( + [ + handler(str), + core_schema.no_info_plain_validator_function(cls.validate), + ] + ) @lru_cache() -@validate_arguments +@validate_call def formatteddatetime( date_format: Optional[str] = None, timezone_treatment: Literal["forbid", "permit", "require"] = "permit", @@ -550,7 +585,7 @@ def formatteddatetime( @lru_cache() -@validate_arguments +@validate_call def formattedtime( time_format: Optional[str] = None, timezone_treatment: Literal["forbid", "permit", "require"] = "permit", @@ -612,13 +647,20 @@ def validate(cls, value: Union[dt.date, str]) -> Optional[dt.date]: return value @classmethod - def __get_validators__(cls) -> Iterator[classmethod]: + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> CoreSchema: """Gets all validators""" - yield cls.validate # type: ignore + return core_schema.chain_schema( + [ + handler(dt.date), + core_schema.no_info_plain_validator_function(cls.validate), + ] + ) # @lru_cache() -@validate_arguments +@validate_call def reportingperiod( reporting_period_type: Literal["start", "end"], date_format: Optional[str] = "%Y-%m-%d" ) -> type[ReportingPeriod]: @@ -632,11 +674,12 @@ def reportingperiod( return type("ReportingPeriod", (ReportingPeriod, *ReportingPeriod.__bases__), dict_) +# TODO - refactor this as it won't work in Pyndatic V2 @lru_cache() -@validate_arguments +@validate_call def alphanumeric( - min_digits: types.NonNegativeInt = 1, - max_digits: types.PositiveInt = 1, + min_digits: types.NonNegativeInt = 1, # pylint: disable=E1101 + max_digits: types.PositiveInt = 1, # pylint: disable=E1101 ) -> type[_SimpleRegexValidator]: """Return a regex-validated class which will ensure that passed numbers are alphanumeric. @@ -651,7 +694,7 @@ def alphanumeric( pattern_str = f"{an_group_str}{{{min_digits},{max_digits}}}" dict_ = _SimpleRegexValidator.__dict__.copy() - dict_["regex"] = re.compile(f"^{pattern_str}$") + dict_["pattern"] = re.compile(f"^{pattern_str}$") return type( type_name, @@ -660,11 +703,12 @@ def alphanumeric( ) +# TODO - refactor this as it won't work in Pyndatic V2 @lru_cache() -@validate_arguments +@validate_call def identifier( - min_digits: types.NonNegativeInt = 1, - max_digits: types.PositiveInt = 1, + min_digits: types.NonNegativeInt = 1, # pylint: disable=E1101 + max_digits: types.PositiveInt = 1, # pylint: disable=E1101 ) -> type[_SimpleRegexValidator]: """ Return a regex-validated class which will ensure that @@ -680,7 +724,7 @@ def identifier( pattern_str = rf"{id_group_str}{{{min_digits},{max_digits}}}" dict_ = _SimpleRegexValidator.__dict__.copy() - dict_["regex"] = re.compile(f"^{pattern_str}$") + dict_["pattern"] = re.compile(f"^{pattern_str}$") return type( type_name, diff --git a/src/dve/metadata_parser/function_library.py b/src/dve/metadata_parser/function_library.py index 0e0e5af..87d0dd4 100644 --- a/src/dve/metadata_parser/function_library.py +++ b/src/dve/metadata_parser/function_library.py @@ -31,7 +31,7 @@ def inner(value, *args, **kwargs): # demo function @_nullcheck -@pydantic.validate_arguments +@pydantic.validate_call def normalise(value, capitalize: bool = False): # pragma: no cover """Normalises a string by capitalising it""" if capitalize: @@ -48,7 +48,7 @@ def exclude_word(value, word: str): @_nullcheck -@pydantic.validate_arguments +@pydantic.validate_call def split(value, split_on: str, keep: int = 0): """Splits a string on a given delimiter and keeps only the value at the given index defaults to 0 diff --git a/src/dve/metadata_parser/function_wrapper.py b/src/dve/metadata_parser/function_wrapper.py index eb69546..cea12c0 100644 --- a/src/dve/metadata_parser/function_wrapper.py +++ b/src/dve/metadata_parser/function_wrapper.py @@ -9,13 +9,14 @@ from dve.metadata_parser import exc PydanticCompatible = Callable[ - [Any, dict[str, Any], pydantic.fields.ModelField, pydantic.BaseConfig], Any + [Any, dict[str, Any], pydantic.fields.FieldInfo, pydantic.ConfigDict], # pylint: disable=E1101 + Any, ] """Function Compatable with pydantic Args: value (Any): Value to be validated values (dict[str, Any]): dict of previously validated fields - field (pydantic.fields.ModelField): field object containing field name and type + field (pydantic.fields.FieldInfo): field object containing field name and type config (pydantic.BaseConfig): the config that determines things like aliases """ @@ -24,21 +25,21 @@ def error_handler( error_type: Union[type[Exception], type[Warning]], error_message: str, - field: pydantic.fields.ModelField, + field: pydantic.fields.FieldInfo, # pylint: disable=E1101 ): """Determines whether to raise an error or warning based on error_type Args: error_type (Union[type[Exception], type[Warning]]): type of error to raise error_message (str): message to apply - field (pydantic.fields.ModelField): field that caused the error to be raised + field (pydantic.fields.FieldInfo): field that caused the error to be raised Raises: error_type """ if issubclass(error_type, exc.LocWarning): - warnings.warn(error_type(msg=error_message, loc=field.name)) + warnings.warn(error_type(msg=error_message, loc=field.title)) elif issubclass(error_type, Warning): warnings.warn(error_message, error_type) else: @@ -90,15 +91,15 @@ def wrapper( Returns: Callable: wrapped function with call signature: - (value: Any, values: dict, field: ModelField, config: BaseConfig) -> Any + (value: Any, values: dict, field: FieldInfo, config: BaseConfig) -> Any """ def inner( value: Any, values: dict[str, Any], - field: pydantic.fields.ModelField, # pylint: disable=unused-argument - config: pydantic.BaseConfig, # pylint: disable=unused-argument + field: pydantic.fields.FieldInfo, # pylint: disable=unused-argument,E1101 + config: pydantic.ConfigDict, # pylint: disable=unused-argument ) -> Any: fields = [values.get(name) for name in field_names] result = None @@ -120,7 +121,7 @@ def inner( return wrapper -validator_args = pydantic.validator.__kwdefaults__.copy() +validator_args = pydantic.field_validator.__kwdefaults__.copy() # type: ignore def create_validator( @@ -153,7 +154,6 @@ def create_validator( always: bool = False check_fields: bool = True whole: bool = None - allow_reuse: bool = False function kwargs @@ -176,26 +176,19 @@ def create_validator( **kwargs, )(function) - validator_kwargs.update(allow_reuse=True) - validator = pydantic.validator(field, **validator_kwargs)(wrapped) + validator = pydantic.field_validator(field, **validator_kwargs)(wrapped) return validator -@pydantic.validate_arguments +@pydantic.validate_call def _validator_kwargs( - pre: bool = False, - each_item: bool = False, - always: bool = False, - check_fields: bool = True, - whole: Optional[bool] = None, - allow_reuse: bool = False, + mode: str = "after", + check_fields: bool | None = True, + json_schema_input_type: Any = None, **kwargs, # pylint: disable=unused-argument ): return { - "pre": pre, - "each_item": each_item, - "always": always, + "mode": mode, "check_fields": check_fields, - "whole": whole, - "allow_reuse": allow_reuse, + "json_schema_input_type": json_schema_input_type, } diff --git a/src/dve/metadata_parser/model_generator.py b/src/dve/metadata_parser/model_generator.py index 2706458..892fbe1 100644 --- a/src/dve/metadata_parser/model_generator.py +++ b/src/dve/metadata_parser/model_generator.py @@ -37,7 +37,7 @@ """ -@pyd.validate_arguments +@pyd.validate_call def constr( *, strip_whitespace: bool = False, @@ -45,7 +45,6 @@ def constr( strict: bool = False, min_length: Optional[int] = None, max_length: Optional[int] = None, - curtail_length: Optional[int] = None, regex: Optional[str] = None, ): """Wrapper around constr to enable argument validation""" @@ -55,16 +54,15 @@ def constr( strict=strict, min_length=min_length, # type: ignore max_length=max_length, # type: ignore - curtail_length=curtail_length, # type: ignore - regex=regex, # type: ignore + pattern=regex, # type: ignore ) STR_TO_PY_MAPPING: Mapping[str, FieldTypeOption] = { "constr": constr, - "conint": pyd.validate_arguments(pyd.conint), - "condate": pyd.validate_arguments(pyd.condate), - "condecimal": pyd.validate_arguments(pyd.condecimal), + "conint": pyd.validate_call(pyd.conint), + "condate": pyd.validate_call(pyd.condate), + "condecimal": pyd.validate_call(pyd.condecimal), "postcode": domain_types.postcode, "nhsnumber": domain_types.NHSNumber, "permissivenhsno": domain_types.permissive_nhs_number(), @@ -89,7 +87,7 @@ def __init__(self, contract_contents: dict[str, Any], type_map: Optional[dict] = @abstractmethod def generate_models( self, additional_validators: Optional[dict] = None - ) -> dict[str, pyd.main.ModelMetaclass]: + ) -> dict[str, pyd.BaseModel]: """Generates models from the instance schema. Args: @@ -112,7 +110,7 @@ def __init__(self, contract_contents: dict[str, Any], type_map: Optional[dict] = def generate_models( self, additional_validators: Optional[dict] = None - ) -> dict[str, pyd.main.ModelMetaclass]: + ) -> dict[str, pyd.BaseModel]: """Generates pydantic models from a loaded json file""" if additional_validators: warnings.warn("Ignoring additional validator functions") diff --git a/src/dve/metadata_parser/models.py b/src/dve/metadata_parser/models.py index 1d64160..49e2386 100644 --- a/src/dve/metadata_parser/models.py +++ b/src/dve/metadata_parser/models.py @@ -5,11 +5,11 @@ import warnings from collections import Counter from collections.abc import Mapping, MutableMapping -from typing import Any, Optional, Union +from typing import Annotated, Any, Optional, Union import pydantic as pyd -from pydantic import BaseModel, Field, root_validator, validator -from typing_extensions import Literal +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator, model_validator +from typing_extensions import Literal, get_origin from dve.metadata_parser import exc, function_library from dve.metadata_parser.function_wrapper import create_validator @@ -27,7 +27,7 @@ """An alias for a field.""" EntityName = str """The name of an entity.""" -PydanticType = Union[type, pyd.main.ModelMetaclass] +PydanticType = Union[type, pyd.BaseModel] """A pydantic-appropriate type.""" ValidatorName = str """The name of a validator.""" @@ -63,19 +63,19 @@ class ValidationFunctionSpecification(BaseModel): # type: ignore kwargs_: dict[str, Any] = Field(default_factory=dict, alias="kwargs") """Keyword arguments for the validation function.""" - @validator("name", allow_reuse=True) - def validate_name(cls, value: str) -> str: + @field_validator("name") + def validate_name(cls, value: str, info: ValidationInfo) -> str: # pylint: disable=W0613 """Ensure that the name exists in the function library.""" if not hasattr(function_library, value): raise ValueError(f"Function {value!r} not available in function library") return value - @validator("error_message", allow_reuse=True) - def validate_error_message(cls, value: str, values: dict[str, Any]) -> str: + @field_validator("error_message") + def validate_error_message(cls, value: str, info: ValidationInfo) -> str: """set a default error message if one is not available.""" if value: return value - name: str = values["name"] + name: str = info.data["name"] return f"{name} failed" def get_field_validator(self, field_name: str, **extra_kwargs: Any) -> classmethod: @@ -128,12 +128,15 @@ class FieldSpecification(BaseModel): functions: list[ValidationFunctionSpecification] = Field(default_factory=list) """Validation functions to be applied to the type.""" - @root_validator(allow_reuse=True) - def ensure_one_type_spec_method(cls, values: dict[str, Any]) -> dict[str, Any]: + model_config = {"validate_assignment": True} + + @model_validator(mode="after") # type: ignore + @classmethod + def ensure_one_type_spec_method(cls, field_spec) -> dict[str, Any]: """Ensure that exactly one of 'type', 'model' and 'callable' was specified.""" - has_type = bool(values.get("type_")) - has_model = bool(values.get("model")) - has_callable = bool(values.get("callable")) + has_type = bool(field_spec.type_) + has_model = bool(field_spec.model) + has_callable = bool(field_spec.callable) n_specified = sum((has_type, has_model, has_callable)) failure_messages = [ @@ -156,21 +159,21 @@ def ensure_one_type_spec_method(cls, values: dict[str, Any]) -> dict[str, Any]: failure_messages.append(f"Got {supplied}") raise ValueError(" ".join(failure_messages)) - if not has_callable and values.get("constraints"): + if not has_callable and field_spec.constraints: warnings.warn( "'constraints' only used when field specification uses 'callable'", category=UnusedConstraints, ) - return values + return field_spec - @validator("default", allow_reuse=True) - def validate_default(cls, value: Any, values: dict[str, Any]) -> Any: + @field_validator("default") + def validate_default(cls, value: Any, info: ValidationInfo) -> Any: """Validate that 'default' is aligned with 'is_array'.""" if value is None: return value - is_array = bool(values.get("is_array")) + is_array = bool(info.data.get("is_array")) if is_array: if not isinstance(value, list): warnings.warn( @@ -212,7 +215,7 @@ def get_type_and_validators( self, field_name: str, *type_mappings: Mapping[TypeName, FieldTypeOption], - schemas: Optional[dict[EntityName, pyd.main.ModelMetaclass]] = None, + schemas: Optional[dict[EntityName, pyd.BaseModel]] = None, is_mandatory: bool = False, ) -> tuple[PydanticType, Default, Validators]: """Get the type, default value, and validators for the specification.""" @@ -223,12 +226,14 @@ def get_type_and_validators( possible_python_type = chain_get(self.type_, *type_mappings, pyd, dt, __builtins__) if isinstance(possible_python_type, type): python_type = possible_python_type + elif get_origin(possible_python_type) is Annotated: + python_type = possible_python_type # type: ignore elif hasattr(possible_python_type, "get_type_and_validators"): possible_python_type: "FieldSpecification" # type: ignore nested_vals = possible_python_type.get_type_and_validators( # type: ignore - field_name, *type_mappings, schemas=schemas, is_mandatory=False + field_name, *type_mappings, schemas=schemas, is_mandatory=is_mandatory ) - python_type, nested_default, nested_validators = nested_vals + python_type, nested_default, nested_validators = nested_vals # type: ignore if nested_validators and self.is_array: # Need to work out how to hook into the validators and update @@ -248,7 +253,7 @@ def get_type_and_validators( if not schemas: raise ValueError("Type should be model, but `schemas` not passed") try: - python_type = schemas[self.model] + python_type = schemas[self.model] # type: ignore except KeyError as err: raise ValueError( f"Type should be model {self.model!r} but this is not in `schemas`" @@ -262,15 +267,20 @@ def get_type_and_validators( raise ValueError("No field type set") default = default or (... if is_mandatory else None) + if self.is_array: python_type = list[python_type] # type: ignore + + if not is_mandatory: + python_type = Optional[python_type] # type: ignore + return python_type, default, validators class EntitySpecification(BaseModel): """Configuration options for an entity.""" - fields: dict[FieldName, FieldSpecification] + fields: Annotated[dict[FieldName, FieldSpecification], Field(validate_default=True)] """ A mapping of field names to their Python types. These will either be strings representing Python types (if there are no argumements to the type), @@ -282,9 +292,11 @@ class EntitySpecification(BaseModel): mandatory_fields: list[FieldName] = Field(default_factory=list) """An array of field names which should be considered mandatory.""" - @validator("fields", pre=True, allow_reuse=True) + @field_validator("fields", mode="before") def validate_fields( - cls, value: dict[FieldName, Union[TypeName, FieldSpecification]] + cls, + value: dict[FieldName, Union[TypeName, FieldSpecification]], + info: ValidationInfo, # pylint: disable=W0613 ) -> dict[FieldName, FieldSpecification]: """Convert bare string fields to field specifications.""" for key in value: @@ -294,9 +306,9 @@ def validate_fields( return value # type: ignore - @validator("aliases", allow_reuse=True) + @field_validator("aliases") def validate_aliases( - cls, value: dict[FieldName, FieldAlias], values: dict[str, Any] + cls, value: dict[FieldName, FieldAlias], info: ValidationInfo ) -> dict[FieldName, FieldAlias]: """Ensure that 'aliases' is aligned with 'fields'.""" # Check that aliases are not given more than once @@ -316,7 +328,7 @@ def validate_aliases( + f"more than once: {multiple_occurrences}" ) # And warn when unnecessary aliases were given. - field_names: set[FieldName] = set(values["fields"].keys()) + field_names: set[FieldName] = set(info.data["fields"].keys()) missing_fields = set(value.keys()) - field_names if missing_fields: warnings.warn( @@ -327,15 +339,15 @@ def validate_aliases( return value - @validator("mandatory_fields", allow_reuse=True) + @field_validator("mandatory_fields") def validate_mandatory_fields( - cls, value: list[FieldName], values: dict[str, Any] + cls, value: list[FieldName], info: ValidationInfo ) -> list[FieldName]: """Ensure that 'mandatory_fields' is aligned with 'fields'.""" if not value: return value - field_names: set[FieldName] = set(values["fields"].keys()) + field_names: set[FieldName] = set(info.data["fields"].keys()) missing_fields = set(value) - field_names if missing_fields: raise ValueError( @@ -349,8 +361,8 @@ def as_model( self, model_name: str, *type_mappings: Mapping[TypeName, FieldTypeOption], - schemas: Optional[dict[EntityName, pyd.main.ModelMetaclass]] = None, - ) -> pyd.main.ModelMetaclass: + schemas: Optional[dict[EntityName, pyd.BaseModel]] = None, + ) -> pyd.BaseModel: """Get the pydantic model from an entity definition.""" validators = {} pyd_fields = {} @@ -365,18 +377,12 @@ def as_model( pyd_fields[field_name] = (python_type, default) validators.update(field_validators) - class Config(pyd.BaseConfig): - """Model configuration.""" - - fields = self.aliases # type: ignore - anystr_strip_whitespace = True - allow_population_by_field_name = True - extra = pyd.Extra.ignore - return pyd.create_model( # type: ignore model_name, **pyd_fields, - __config__=Config, # type: ignore + __config__=ConfigDict( + str_strip_whitespace=True, populate_by_name=True, extra="ignore" + ), # type: ignore __validators__=validators, ) @@ -395,9 +401,9 @@ class DatasetSpecification(BaseModel): def load_models( self, *type_mappings: Mapping[TypeName, FieldTypeOption], - ) -> dict[EntityName, pyd.main.ModelMetaclass]: + ) -> dict[EntityName, pyd.BaseModel]: """Load the models from the dataset definition.""" - loaded_schemas: dict[EntityName, pyd.main.ModelMetaclass] = {} + loaded_schemas: dict[EntityName, pyd.BaseModel] = {} for model_name, specification in self.schemas.items(): # pylint: disable=E1101 loaded_schemas[model_name] = specification.as_model( model_name, self.types, *type_mappings, schemas=loaded_schemas diff --git a/src/dve/parser/file_handling/implementations/file.py b/src/dve/parser/file_handling/implementations/file.py index 76d8b58..9fb29db 100644 --- a/src/dve/parser/file_handling/implementations/file.py +++ b/src/dve/parser/file_handling/implementations/file.py @@ -2,7 +2,7 @@ import platform import shutil -from collections.abc import Callable, Iterator +from collections.abc import Iterator from pathlib import Path from typing import IO, Any, NoReturn, Optional from urllib.parse import unquote @@ -12,7 +12,7 @@ from dve.parser.exceptions import FileAccessError from dve.parser.file_handling.helpers import parse_uri from dve.parser.file_handling.implementations.base import BaseFilesystemImplementation -from dve.parser.type_hints import URI, NodeType, PathStr, Scheme +from dve.parser.type_hints import URI, NodeType, Scheme FILE_URI_SCHEMES: set[Scheme] = {"file"} """A set of all allowed file URI schemes.""" @@ -179,9 +179,9 @@ def _transfer_resource( parent.mkdir(exist_ok=True) if action == "copy": - func: Callable[[PathStr, PathStr], None] = shutil.copy + func = shutil.copy # type: ignore elif action == "move": - func = shutil.move + func = shutil.move # type: ignore else: # pragma: no cover raise ValueError(f"Unsupported action {action!r}, expected one of: 'copy', 'move'") try: diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py index 234021b..a9be3ff 100644 --- a/src/dve/pipeline/pipeline.py +++ b/src/dve/pipeline/pipeline.py @@ -14,7 +14,7 @@ from uuid import uuid4 import polars as pl -from pydantic import validate_arguments +from pydantic import validate_call import dve.reporting.excel_report as er from dve.common.error_utils import ( @@ -142,7 +142,7 @@ def get_submission_status( return SubmissionStatus() return submission_status - @validate_arguments + @validate_call def _move_submission_to_working_location( self, submission_id: str, @@ -846,7 +846,7 @@ def error_report( summary_dict = { key.replace("_", " ").title(): value - for key, value in submission_info.dict().items() + for key, value in submission_info.model_dump().items() if value is not None and not key.endswith("_updated") } summary_items = er.SummaryItems( diff --git a/src/dve/pipeline/utils.py b/src/dve/pipeline/utils.py index 99f2da7..e6122c2 100644 --- a/src/dve/pipeline/utils.py +++ b/src/dve/pipeline/utils.py @@ -5,7 +5,7 @@ from threading import Lock from typing import Any, Optional -from pydantic.main import ModelMetaclass +from pydantic import BaseModel from pyspark.sql import SparkSession import dve.core_engine.backends.implementations.duckdb # pylint: disable=unused-import @@ -18,7 +18,7 @@ from dve.metadata_parser.model_generator import JSONtoPyd Dataset = dict[SchemaName, _ModelConfig] -_configs: dict[str, tuple[dict[str, ModelMetaclass], V1EngineConfig, Dataset]] = {} +_configs: dict[str, tuple[dict[str, BaseModel], V1EngineConfig, Dataset]] = {} locks = Lock() logger = get_logger(__name__) @@ -27,7 +27,7 @@ def load_config( dataset_id: str, file_uri: URI, -) -> tuple[dict[SchemaName, ModelMetaclass], V1EngineConfig, dict[SchemaName, _ModelConfig]]: +) -> tuple[dict[SchemaName, BaseModel], V1EngineConfig, dict[SchemaName, _ModelConfig]]: """Loads the configuration for a given dataset""" if dataset_id in _configs: return _configs[dataset_id] diff --git a/tests/features/steps/steps_post_pipeline.py b/tests/features/steps/steps_post_pipeline.py index be679ba..906445e 100644 --- a/tests/features/steps/steps_post_pipeline.py +++ b/tests/features/steps/steps_post_pipeline.py @@ -108,7 +108,7 @@ def check_stats_record(context): record: Dict[str, str] = row.as_dict() expected[record["parameter"]] = int(record["value"]) stats = ( - get_pipeline(context)._audit_tables.get_submission_statistics(sub_info.submission_id).dict() + get_pipeline(context)._audit_tables.get_submission_statistics(sub_info.submission_id).model_dump() ) assert all([val == stats.get(fld) for fld, val in expected.items()]) diff --git a/tests/fixtures.py b/tests/fixtures.py index 457cf42..fa79eec 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -12,9 +12,6 @@ from moto import mock_s3 # type: ignore from pyspark.sql import SparkSession -from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import ( - PYTHON_TYPE_TO_DUCKDB_TYPE, -) from dve.parser.file_handling.implementations import DBFSFilesystemImplementation from dve.parser.file_handling.service import ( add_implementation, diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py index 5daa471..2ae7a65 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_audit_ddb.py @@ -344,11 +344,11 @@ def test_get_error_report_submissions(ddb_audit_manager_threaded: DDBAuditingMan expected = [ SubmissionInfo( submission_id=sub_1.submission_id, - **{fld: val for fld, val in sub_1.dict().items() if fld != "submission_id"}, + **{fld: val for fld, val in sub_1.model_dump().items() if fld != "submission_id"}, ), SubmissionInfo( submission_id=sub_3.submission_id, - **{fld: val for fld, val in sub_3.dict().items() if fld != "submission_id"}, + **{fld: val for fld, val in sub_3.model_dump().items() if fld != "submission_id"}, ), ] assert len(processed) == 2 @@ -420,8 +420,8 @@ def test_get_submission_status(ddb_audit_manager_threaded: DDBAuditingManager): ] ) aud.add_submission_statistics_records([ - SubmissionStatisticsRecord(submission_id=sub_1.submission_id, record_count=5, number_record_rejections=2, number_warnings=3), - SubmissionStatisticsRecord(submission_id=sub_4.submission_id, record_count=20, number_record_rejections=0, number_warnings=1) + SubmissionStatisticsRecord(submission_id=sub_1.submission_id, record_count=5, number_submission_rejections=0, number_record_rejections=2, number_warnings=3), + SubmissionStatisticsRecord(submission_id=sub_4.submission_id, record_count=20, number_submission_rejections=0, number_record_rejections=0, number_warnings=1) ]) sub_stats_1 = ddb_audit_manager_threaded.get_submission_status(sub_1.submission_id) diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py index 74edce9..2019a66 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py @@ -96,8 +96,8 @@ def test_duckdb_data_contract_csv(temp_csv_file): entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(uri.as_posix()), entities, entity_locations, dc_meta) rel: DuckDBPyRelation = entities.get("test_ds") expected_schema = { - fld.name: str(get_duckdb_type_from_annotation(fld.annotation)) - for fld in mdl.__fields__.values() + name: str(get_duckdb_type_from_annotation(fld.annotation)) + for name, fld in mdl.model_fields.items() } expected_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int) assert dict(zip(rel.columns, rel.dtypes)) == expected_schema @@ -196,13 +196,13 @@ def test_duckdb_data_contract_xml(temp_xml_file): entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(uri.as_posix()), entities, entity_locations, dc_meta) header_rel: DuckDBPyRelation = entities.get("test_header") header_expected_schema: Dict[str, DuckDBPyType] = { - fld.name: get_duckdb_type_from_annotation(fld.type_) - for fld in header_model.__fields__.values() + name: get_duckdb_type_from_annotation(fld.annotation) + for name, fld in header_model.model_fields.items() } header_expected_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int) class_data_expected_schema: Dict[str, DuckDBPyType] = { - fld.name: get_duckdb_type_from_annotation(fld.type_) - for fld in class_model.__fields__.values() + name: get_duckdb_type_from_annotation(fld.annotation) + for name, fld in class_model.model_fields.items() } class_data_expected_schema[RECORD_INDEX_COLUMN_NAME] = get_duckdb_type_from_annotation(int) class_data_rel: DuckDBPyRelation = entities.get("test_class_info") diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py index 7926ad9..4eefe05 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py @@ -231,9 +231,9 @@ def test_get_duckdb_cast_statement_from_annotation(field_name, field_type, cast_ def test_use_cast_statements(casting_test_table): _, conn = casting_test_table test_rel = conn.sql("SELECT * from test_casting") - casting_statements = [ f"{get_duckdb_cast_statement_from_annotation(fld.name, fld.annotation)} as {fld.name}" for fld in CastingRecord.__fields__.values()] + casting_statements = [ f"{get_duckdb_cast_statement_from_annotation(name, fld.annotation)} as {name}" for name, fld in CastingRecord.model_fields.items()] test_rel = test_rel.project(",".join(casting_statements)) - assert dict(zip(test_rel.columns, test_rel.dtypes)) == {fld.name: get_duckdb_type_from_annotation(fld.annotation) for fld in CastingRecord.__fields__.values()} + assert dict(zip(test_rel.columns, test_rel.dtypes)) == {name: get_duckdb_type_from_annotation(fld.annotation) for name, fld in CastingRecord.model_fields.items()} good_date_rec = test_rel.pl()[0].to_dicts()[0] dodgy_date_rec = test_rel.pl()[1].to_dicts()[0] assert (good_date_rec.get("date_test") and good_date_rec.get("timestamp_test") and diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py index 5d3d3a8..35007a9 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py @@ -69,7 +69,7 @@ def test_column_addition(planets_rel: DuckDBPyRelation): rule = ColumnAddition( entity_name="planets", column_name="literal_one", - expression=1, + expression="1", new_entity_name="added", ) _, success = DUCKDB_STEP_BACKEND.evaluate(entities, config=rule) @@ -87,7 +87,7 @@ def test_column_addition_missing_entity(): rule = ColumnAddition( entity_name="planets", column_name="literal_one", - expression=1, + expression="1", new_entity_name="added", ) messages, success = DUCKDB_STEP_BACKEND.evaluate(entities, config=rule) diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py index 4b328df..9d47dfd 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_audit_spark.py @@ -359,11 +359,11 @@ def test_get_error_report_submissions(spark_audit_manager: SparkAuditingManager) expected = [ SubmissionInfo( submission_id=sub_1.submission_id, - **{fld: val for fld, val in sub_1.dict().items() if fld != "submission_id"}, + **{fld: val for fld, val in sub_1.model_dump().items() if fld != "submission_id"}, ), SubmissionInfo( submission_id=sub_3.submission_id, - **{fld: val for fld, val in sub_3.dict().items() if fld != "submission_id"}, + **{fld: val for fld, val in sub_3.model_dump().items() if fld != "submission_id"}, ), ] assert len(processed) == 2 @@ -435,8 +435,8 @@ def test_get_submission_status(spark_audit_manager: SparkAuditingManager): ] ) aud.add_submission_statistics_records([ - SubmissionStatisticsRecord(submission_id=sub_1.submission_id, record_count=5, number_record_rejections=2, number_warnings=3), - SubmissionStatisticsRecord(submission_id=sub_4.submission_id, record_count=20, number_record_rejections=0, number_warnings=1) + SubmissionStatisticsRecord(submission_id=sub_1.submission_id, record_count=5, number_submission_rejections=0, number_record_rejections=2, number_warnings=3), + SubmissionStatisticsRecord(submission_id=sub_4.submission_id, record_count=20, number_submission_rejections=0, number_record_rejections=0, number_warnings=1) ]) sub_stats_1 = aud.get_submission_status(sub_1.submission_id) diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py index e0cbf90..673e611 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py @@ -59,7 +59,7 @@ def test_column_addition(planets_df: DataFrame): rule = ColumnAddition( entity_name="planets", column_name="literal_one", - expression=1, + expression="1", new_entity_name="added", ) _, success = SPARK_STEP_BACKEND.evaluate(entities, config=rule) @@ -77,7 +77,7 @@ def test_column_addition_missing_entity(): rule = ColumnAddition( entity_name="planets", column_name="literal_one", - expression=1, + expression="1", new_entity_name="added", ) messages, success = SPARK_STEP_BACKEND.evaluate(entities, config=rule) diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py index 3fcf5d8..e7a37eb 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py @@ -311,9 +311,9 @@ def test_get_spark_cast_statement_from_annotation(field_name, field_type, expres def test_use_cast_statements(spark, casting_dataframe): - casting_statements = [ get_spark_cast_statement_from_annotation(fld.name, fld.annotation).alias(fld.name) for fld in CastingRecord.__fields__.values()] + casting_statements = [ get_spark_cast_statement_from_annotation(name, fld.annotation).alias(name) for name, fld in CastingRecord.model_fields.items()] cast_df = casting_dataframe.select(*casting_statements) - assert {fld.name: fld.dataType for fld in cast_df.schema} == {fld.name: get_type_from_annotation(fld.annotation) for fld in CastingRecord.__fields__.values()} + assert {fld.name: fld.dataType for fld in cast_df.schema} == {name: get_type_from_annotation(fld.annotation) for name, fld in CastingRecord.model_fields.items()} good_date_rec, dodgy_date_rec = (rw.asDict(True) for rw in cast_df.collect()) assert (good_date_rec.get("date_test") and good_date_rec.get("timestamp_test") and good_date_rec.get("basic_model",{}).get("date_field") and good_date_rec.get("basic_model",{}).get("timestamp_field") diff --git a/tests/test_core_engine/test_backends/test_readers/test_csv.py b/tests/test_core_engine/test_backends/test_readers/test_csv.py index f47ce6b..f2e2c8d 100644 --- a/tests/test_core_engine/test_backends/test_readers/test_csv.py +++ b/tests/test_core_engine/test_backends/test_readers/test_csv.py @@ -159,7 +159,7 @@ def test_csv_file_get_subset( parsed = {row["planet"]: row for row in results} # Keep only keys in the subset from the source - subset_keys = set(PlanetsSubset.__fields__.keys()) + subset_keys = set(PlanetsSubset.model_fields.keys()) for data in planet_data.values(): to_pop = set(data.keys()) - subset_keys - {RECORD_INDEX_COLUMN_NAME} for key in to_pop: @@ -181,7 +181,7 @@ def test_csv_file_get_subset_add_missing( parsed = {row["planet"]: row for row in results} # Keep only keys in the subset from the source - subset_keys = set(PlanetsSubset.__fields__.keys()) + subset_keys = set(PlanetsSubset.model_fields.keys()) for data in planet_data.values(): to_pop = set(data.keys()) - subset_keys - {RECORD_INDEX_COLUMN_NAME} for key in to_pop: diff --git a/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_csv.py b/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_csv.py index b3178cd..8bc9775 100644 --- a/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_csv.py +++ b/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_csv.py @@ -121,8 +121,8 @@ def test_ddb_csv_reader_cast(self, temp_csv_file): reader = DuckDBCSVReader(header=True, delim=",", connection=duckdb.connect()) rel: DuckDBPyRelation = reader.read_to_entity_type(DuckDBPyRelation, str(uri), "test", mdl) expected_dtypes = {**{ - fld.name: str(get_duckdb_type_from_annotation(fld.annotation)) - for fld in mdl.__fields__.values() + fld_name: str(get_duckdb_type_from_annotation(fld_info.annotation)) + for fld_name, fld_info in mdl.model_fields.items() }, RECORD_INDEX_COLUMN_NAME: get_duckdb_type_from_annotation(int)} expected_data = [(*rw, idx) for idx, rw in enumerate(data, start=1)] assert rel.columns == header.split(",") + [RECORD_INDEX_COLUMN_NAME] diff --git a/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_json.py b/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_json.py index 87a90e8..ae8e4ad 100644 --- a/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_json.py +++ b/tests/test_core_engine/test_backends/test_readers/test_duckdb/test_ddb_json.py @@ -56,7 +56,7 @@ class SimpleModel(BaseModel): def test_ddb_json_reader_all_str(temp_json_file): uri, data, mdl = temp_json_file - expected_fields = [fld for fld in mdl.__fields__] + expected_fields = [fld for fld in mdl.model_fields] reader = DuckDBJSONReader() rel: DuckDBPyRelation = reader.read_to_entity_type( DuckDBPyRelation, uri.as_posix(), "test", stringify_model(mdl) @@ -68,14 +68,14 @@ def test_ddb_json_reader_all_str(temp_json_file): def test_ddb_json_reader_cast(temp_json_file): uri, data, mdl = temp_json_file - expected_fields = [fld for fld in mdl.__fields__] + expected_fields = [fld for fld in mdl.model_fields] reader = DuckDBJSONReader() rel: DuckDBPyRelation = reader.read_to_entity_type(DuckDBPyRelation, uri.as_posix(), "test", mdl) assert rel.columns == expected_fields + [RECORD_INDEX_COLUMN_NAME] assert dict(zip(rel.columns, rel.dtypes)) == {**{ - fld.name: str(get_duckdb_type_from_annotation(fld.annotation)) - for fld in mdl.__fields__.values() + name: str(get_duckdb_type_from_annotation(fld.annotation)) + for name, fld in mdl.model_fields.items() }, RECORD_INDEX_COLUMN_NAME: "BIGINT"} assert rel.fetchall() == [(*rw.values(), idx) for idx, rw in enumerate(data, start = 1)] diff --git a/tests/test_core_engine/test_backends/test_readers/test_spark/test_spark_json.py b/tests/test_core_engine/test_backends/test_readers/test_spark/test_spark_json.py index 24674ca..46d4729 100644 --- a/tests/test_core_engine/test_backends/test_readers/test_spark/test_spark_json.py +++ b/tests/test_core_engine/test_backends/test_readers/test_spark/test_spark_json.py @@ -55,7 +55,7 @@ class SimpleModel(BaseModel): def test_spark_json_reader_all_str(temp_json_file): uri, data, mdl = temp_json_file - expected_fields = [fld for fld in mdl.__fields__] + [RECORD_INDEX_COLUMN_NAME] + expected_fields = [fld for fld in mdl.model_fields] + [RECORD_INDEX_COLUMN_NAME] reader = SparkJSONReader() df: DataFrame = reader.read_to_entity_type( DataFrame, uri.as_posix(), "test", stringify_model(mdl) @@ -66,13 +66,13 @@ def test_spark_json_reader_all_str(temp_json_file): def test_spark_json_reader_cast(temp_json_file): uri, data, mdl = temp_json_file - expected_fields = [fld for fld in mdl.__fields__] + [RECORD_INDEX_COLUMN_NAME] + expected_fields = [fld for fld in mdl.model_fields] + [RECORD_INDEX_COLUMN_NAME] reader = SparkJSONReader() df: DataFrame = reader.read_to_entity_type(DataFrame, uri.as_posix(), "test", mdl) assert df.columns == expected_fields - assert df.schema == StructType([StructField(fld.name, get_type_from_annotation(fld.annotation)) - for fld in mdl.__fields__.values()] + [StructField(RECORD_INDEX_COLUMN_NAME, get_type_from_annotation(int))]) + assert df.schema == StructType([StructField(name, get_type_from_annotation(fld.annotation)) + for name, fld in mdl.model_fields.items()] + [StructField(RECORD_INDEX_COLUMN_NAME, get_type_from_annotation(int))]) assert [rw.asDict() for rw in df.collect()] == [{**rw, RECORD_INDEX_COLUMN_NAME: idx} for idx, rw in enumerate(data, start=1)] diff --git a/tests/test_core_engine/test_backends/test_utilities.py b/tests/test_core_engine/test_backends/test_utilities.py new file mode 100644 index 0000000..1c9121f --- /dev/null +++ b/tests/test_core_engine/test_backends/test_utilities.py @@ -0,0 +1,112 @@ +""" +Test backend utility functions +""" + +# pylint: disable=C0115,C0116 + +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +import pytest +from pydantic import BaseModel + +from dve.core_engine.backends.utilities import is_field_complex, stringify_model +from dve.metadata_parser.domain_types import FormattedDatetime + + +def model_to_schema(mdl: BaseModel) -> dict[str, Any]: + schema = {} + for field_name, field_md in mdl.model_fields.items(): + if issubclass(field_md.annotation, BaseModel): + schema[field_name] = model_to_schema(field_md.annotation) + else: + schema[field_name] = field_md.annotation + + return schema + + +class AnotherTestModel(BaseModel): + test_field: str + + +class TestModel(BaseModel): + simple_str: str + simple_int: int + simple_bool: bool + simple_list: list[str] + simple_dict: dict[str, Any] + simple_set: set[str] + simple_tuple: tuple[int, str] + another_model: AnotherTestModel + date_example: date + test_fdatetime: FormattedDatetime + + +class ChildMultiTypeModel(BaseModel): + child_test_int: int + + +class MultiTypeModel(BaseModel): + test_str: str + test_int: int + test_dict: dict[str, int] + test_list: list[int] + test_decimal: Decimal + test_date: date + test_datetime: datetime + test_fdatetime: FormattedDatetime + test_child_model: ChildMultiTypeModel + + +TEST_MODEL = TestModel( + simple_str="abc", + simple_int=123, + simple_bool=True, + simple_list=["apple", "banana", "orange",], + simple_dict={"key1": "abc"}, + simple_set={"a", "b", "c"}, + simple_tuple=(1, "wow"), + another_model=AnotherTestModel(test_field="lemon"), + date_example=date(2026,1,1), + test_fdatetime="2026-01-01T12:30:10", +) + + +@pytest.mark.parametrize( + "field_name, expected", + [ + ("simple_str", False), + ("simple_int", False), + ("simple_bool", False), + ("simple_list", True), + ("simple_dict", True), + ("simple_set", True), + ("simple_tuple", True), + ("another_model", True), + ("test_fdatetime", False), + ] +) +def test_is_field_complex(field_name: str, expected: bool): + _field = TEST_MODEL.model_fields[field_name] + _res = is_field_complex(_field) + assert _res == expected, f"Expected {expected}. Got {_res}." + + +def test_stringify_model(): + expected_dict = { + "test_str": str, + "test_int": str, + "test_dict": dict[str, str], + "test_list": list[str], + "test_decimal": str, + "test_date": str, + "test_datetime": str, + "test_fdatetime": str, + "test_child_model": { + "child_test_int": str + }, + } + serialised_model_result = model_to_schema(stringify_model(MultiTypeModel)) + + assert expected_dict == serialised_model_result diff --git a/tests/test_core_engine/test_message.py b/tests/test_core_engine/test_message.py index 4c092f1..bb0d0d8 100644 --- a/tests/test_core_engine/test_message.py +++ b/tests/test_core_engine/test_message.py @@ -4,8 +4,9 @@ import json from string import ascii_letters from typing import Dict, List, Optional +from typing_extensions import Annotated -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, Field, ValidationError import pytest from dve.core_engine.message import DEFAULT_ERROR_DETAIL, DataContractErrorDetail, FeedbackMessage @@ -140,7 +141,7 @@ def test_from_pydantic_error(): class TestModel(BaseModel): idx: int - other_field: str + other_field: Annotated[str, Field(coerce_numbers_to_str=True)] _bad_value_data = {"idx": "ABC", "other_field": 123} _blank_value_data = {"other_field": "hi"} @@ -178,9 +179,9 @@ def test_from_pydantic_error_custom_error_details(): class TestModel(BaseModel): idx: int str_field: str - date_field: Optional[date] - unimportant_field: Optional[int] - + date_field: Annotated[date, Field(default=None)] + unimportant_field: Annotated[int, Field(default=None)] + custom_error_details: str = """ {"idx": {"Blank": {"error_code": "IDBLANKERRCODE", "error_message": "idx is a mandatory field", @@ -248,7 +249,7 @@ def test_from_pydantic_error_custom_codes_nested(): class LowestModel(BaseModel): nested_field_3: str - test_date: Optional[date] + test_date: Annotated[date, Field(default=None)] class SubTestModel(BaseModel): nested_field_1: int nested_field_2: List[LowestModel] diff --git a/tests/test_core_engine/test_models.py b/tests/test_core_engine/test_models.py index e187de1..d1a55c4 100644 --- a/tests/test_core_engine/test_models.py +++ b/tests/test_core_engine/test_models.py @@ -1,10 +1,11 @@ """Unit tests for core engine models.""" +from datetime import date, datetime from typing import Any, Dict, Tuple from uuid import uuid4 import pytest -from dve.core_engine.models import SubmissionInfo +from dve.core_engine.models import AuditRecord, SubmissionInfo CONSTANT_SUBMISSION_ID = uuid4().hex @@ -116,7 +117,7 @@ def test_submission_info( # pylint: disable=missing-function-docstring actual = SubmissionInfo(**testcase["submitted"]) expected = testcase["expected"] - assert {k: v for k, v in actual.dict().items() if k not in ignore} == expected + assert {k: v for k, v in actual.model_dump().items() if k not in ignore} == expected assert actual.file_name_with_ext == f"{expected['file_name']}.{expected['file_extension']}" @@ -128,3 +129,23 @@ def test_submission_info_eq(): # pylint: disable=missing-function-docstring "file_extension": "csv", } assert SubmissionInfo(**data) == SubmissionInfo(**data) # type: ignore + + +class TestAuditRecord: + submission_id = uuid4().hex + + def test_audit_record_just_time_updated(self): + data = { + "submission_id": self.submission_id, + "time_updated": datetime(2025,12,1,12,30,10) + } + + assert AuditRecord(**data).date_updated == date(2025,12,1) + + def test_audit_record_date_updated(self): + data = { + "submission_id": self.submission_id, + "date_updated": date(2025,12,1) + } + + assert AuditRecord(**data).date_updated == date.today() diff --git a/tests/test_model_generation/test_domain_types.py b/tests/test_model_generation/test_domain_types.py index 56cf3f9..8917bac 100644 --- a/tests/test_model_generation/test_domain_types.py +++ b/tests/test_model_generation/test_domain_types.py @@ -15,10 +15,10 @@ class ATestModel(BaseModel): - nhsnumber: Optional[hct.NHSNumber] - postcode: Optional[hct.Postcode] - org_id: Optional[hct.OrgID] - nhsnumber2: Optional[hct.permissive_nhs_number()] + nhsnumber: Optional[hct.NHSNumber] = None + postcode: Optional[hct.Postcode] = None + org_id: Optional[hct.OrgID] = None + nhsnumber2: Optional[hct.permissive_nhs_number()] = None class DatetimeModel(BaseModel): @@ -26,8 +26,8 @@ class DatetimeModel(BaseModel): class ReportingPeriodModel(BaseModel): - reporting_period_start: Optional[hct.reportingperiod(reporting_period_type="start")] - reporting_period_end: Optional[hct.reportingperiod(reporting_period_type="end")] + reporting_period_start: Optional[hct.reportingperiod(reporting_period_type="start")] = None + reporting_period_end: Optional[hct.reportingperiod(reporting_period_type="end")] = None @pytest.mark.parametrize( @@ -256,9 +256,9 @@ def test_formatteddatetime_in_model_raises(datetime_to_validate: Union[str, dt.d class DateModel(BaseModel): """Model for testing FormattedDate.""" - formatted_date: Optional[hct.conformatteddate("%Y-%m-%d")] - formatted_date_constrained: Optional[hct.conformatteddate("%Y-%m-%d", ge="1970-01-01")] - formatted_date_constrained2: Optional[hct.conformatteddate("%Y-%m-%d", lt="1970-01-01")] + formatted_date: Optional[hct.conformatteddate("%Y-%m-%d")] = None + formatted_date_constrained: Optional[hct.conformatteddate("%Y-%m-%d", ge="1970-01-01")] = None + formatted_date_constrained2: Optional[hct.conformatteddate("%Y-%m-%d", lt="1970-01-01")] = None @pytest.mark.parametrize( diff --git a/tests/test_pipeline/test_duckdb_pipeline.py b/tests/test_pipeline/test_duckdb_pipeline.py index aa65516..4e41af0 100644 --- a/tests/test_pipeline/test_duckdb_pipeline.py +++ b/tests/test_pipeline/test_duckdb_pipeline.py @@ -250,7 +250,7 @@ def test_get_submission_status(temp_ddb_conn): ] ) aud.add_submission_statistics_records([ - SubmissionStatisticsRecord(submission_id=sub_one.submission_id, record_count=5, number_record_rejections=2, number_warnings=3), + SubmissionStatisticsRecord(submission_id=sub_one.submission_id, record_count=5, number_submission_rejections=0, number_record_rejections=2, number_warnings=3), ]) sub_stats_one = dve_pipeline.get_submission_status("test", sub_one.submission_id) diff --git a/tests/test_pipeline/test_spark_pipeline.py b/tests/test_pipeline/test_spark_pipeline.py index 960ad50..dd28e26 100644 --- a/tests/test_pipeline/test_spark_pipeline.py +++ b/tests/test_pipeline/test_spark_pipeline.py @@ -166,7 +166,7 @@ def test_apply_data_contract_failed( # pylint: disable=redefined-outer-name "Key": "", "FailureType": "record", "Status": "error", - "ErrorType": "value_error.any_str.max_length", + "ErrorType": "string_too_long", "ErrorLocation": "planet", "ErrorMessage": "is invalid", "ErrorCode": "BadValue", @@ -180,7 +180,7 @@ def test_apply_data_contract_failed( # pylint: disable=redefined-outer-name "Key": "", "FailureType": "record", "Status": "error", - "ErrorType": "value_error.number.not_ge", + "ErrorType": "greater_than_equal", "ErrorLocation": "numberOfMoons", "ErrorMessage": "is invalid", "ErrorCode": "BadValue", @@ -194,7 +194,7 @@ def test_apply_data_contract_failed( # pylint: disable=redefined-outer-name "Key": "", "FailureType": "record", "Status": "error", - "ErrorType": "type_error.bool", + "ErrorType": "bool_parsing", "ErrorLocation": "hasGlobalMagneticField", "ErrorMessage": "is invalid", "ErrorCode": "BadValue", @@ -421,7 +421,7 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out "number_warnings": 0, } - sub_stats = stats.dict() + sub_stats = stats.model_dump() if stats else {} assert all([expected.get(key) == sub_stats.get(key) for key in expected]) @@ -612,7 +612,7 @@ def test_get_submission_status(spark, spark_test_database): ] ) audit_manager.add_submission_statistics_records([ - SubmissionStatisticsRecord(submission_id=sub_one.submission_id, record_count=5, number_record_rejections=2, number_warnings=3), + SubmissionStatisticsRecord(submission_id=sub_one.submission_id, record_count=5, number_submission_rejections=0, number_record_rejections=2, number_warnings=3), ]) sub_stats_one = dve_pipeline.get_submission_status("test", sub_one.submission_id) diff --git a/zensical.toml b/zensical.toml index 255a771..064cb26 100644 --- a/zensical.toml +++ b/zensical.toml @@ -67,7 +67,8 @@ nav = [ {"Implementing a new backend" = "advanced_guidance/new_backend.md"}, {"Dischema Language Server" = "advanced_guidance/json_schemas.md"}, ]}, - ]} + ]}, + {"FAQ" = "faq.md"} ] extra_css = ["assets/stylesheets/extra.css"] # extra_javascript = ["assets/javascript/extra.js"]