diff --git a/changes/4101.changed.md b/changes/4101.changed.md new file mode 100644 index 0000000000..003f2e85c8 --- /dev/null +++ b/changes/4101.changed.md @@ -0,0 +1,40 @@ +Gave `zarr.config` a statically-typed, dataclass-backed representation. +`zarr.config` now provides precise static types for attribute access +(`zarr.config.array.order`) and for the dotted-string API +(`zarr.config.get("array.order")`), and validates configuration *keys* at +runtime with suggestions for typos. External values are validated when the +configuration is created; for example, `ZARR_ARRAY__ORDER=Q` raises an error. +Programmatic `config.set` continues to defer value validation to use sites. +`donfig` is retained as the reader for environment +variables (`ZARR_FOO__BAR`) and YAML config files, so their locations and +precedence are unchanged. The string API, `config.set` (permanent and as a +context manager, including cross-thread visibility of permanent overrides), +`config.reset`, `config.enable_gpu`, and the `deprecations` mechanism are all +preserved. The former `Config` class name remains importable from +`zarr.core.config` as an alias of the new manager. +Hyphenated and underscored key spellings remain interchangeable, including in +YAML files, and nested keyword overrides such as `config.set(array__order="F")` +remain supported. The configuration manager can still be deep-copied. + +Environment collection recognizes config names separately from controls such +as `ZARR_CONFIG`, `ZARR_ROOT_CONFIG`, and `ZARR_BENCHMARK_CLEAR_CACHE`. Controls +never become configuration fields. Unknown environment names and YAML keys +warn and are skipped during collection; construction rejects unknown keys and +invalid value types. Custom names remain supported in the open `codecs` namespace. + +Note: `zarr.config.defaults` now returns a nested `dict` directly; donfig +previously returned a one-element `list[dict]`, so callers that used +`config.defaults[0]` must be updated to use `config.defaults`. Subtree reads +such as `config.get("array")` now return a typed settings object rather than a +plain `dict`; it supports attribute access (`config.get("array").order`), item +access (`config.get("array")["order"]`), membership (`"order" in ...`), +iteration, `.keys()`, and `dict(...)`, but is immutable. The `codecs` mapping is +likewise exposed read-only through `zarr.config.codecs`; change codec selections +via `config.set` rather than by mutating the mapping in place. + +A few donfig `Config` methods are not reimplemented: `config.clear()` and +`config.merge()` are gone (use `config.reset()` to restore defaults), and +`config.pprint()` no longer accepts `stream`/`width` arguments. Assigning a whole +structured subtree to a mapping (for example `config.set({"array": {"order": +"F"}})`) is now rejected with an error; set leaf keys such as `array.order` +instead. diff --git a/changes/README.md b/changes/README.md index 889a52baa4..c815c457c3 100644 --- a/changes/README.md +++ b/changes/README.md @@ -6,6 +6,7 @@ Please put a new file in this directory named `xxxx..md`, where - `xxxx` is the pull request number associated with this entry - `` is one of: - feature + - changed - bugfix - doc - removal diff --git a/ci/check_changelog_entries.py b/ci/check_changelog_entries.py index 42d7cc1708..f548401fc0 100644 --- a/ci/check_changelog_entries.py +++ b/ci/check_changelog_entries.py @@ -10,7 +10,7 @@ import sys from pathlib import Path -VALID_CHANGELOG_TYPES = ["feature", "bugfix", "doc", "removal", "misc"] +VALID_CHANGELOG_TYPES = ["feature", "changed", "bugfix", "doc", "removal", "misc"] REPO_ROOT = Path(__file__).parent.parent.resolve() DEFAULT_DIRECTORY = REPO_ROOT / "changes" diff --git a/docs/user-guide/config.md b/docs/user-guide/config.md index d1a70a14b0..4e274ea4bc 100644 --- a/docs/user-guide/config.md +++ b/docs/user-guide/config.md @@ -1,7 +1,8 @@ # Runtime configuration -[`zarr.config`][] is responsible for managing the configuration of zarr and -is based on the [donfig](https://github.com/pytroll/donfig) Python library. +[`zarr.config`][] is a `ZarrConfigManager` instance that manages all runtime +settings for zarr. It provides both typed attribute access and a dotted-string +key API. Configuration values can be set using code like the following: @@ -26,12 +27,42 @@ with zarr.config.set({'array.order': 'F'}): print(zarr.config.get('array.order')) ``` -Alternatively, configuration values can be set using environment variables, e.g. +Alternatively, configuration values can be set using environment variables. +The variable name uses a `ZARR_` prefix, with `__` to denote nesting, e.g. `ZARR_ARRAY__ORDER=F`. -The configuration can also be read from a YAML file in standard locations. -For more information, see the -[donfig documentation](https://donfig.readthedocs.io/en/latest/). +External configuration is processed in two stages. Collection recognizes names +from the configuration schema and a separate set of environment controls: +`ZARR_CONFIG` and `ZARR_ROOT_CONFIG` control file discovery, and +`ZARR_BENCHMARK_CLEAR_CACHE` controls benchmark cache clearing. These controls +are left to their consumers and never become configuration fields. Unrecognized +`ZARR_*` names produce a warning and are ignored. The `codecs` namespace is open, +so environment variables can also select implementations for custom codec names. + +Config creation then validates the collected values against the schema and +applies them to the defaults. Invalid values raise an error identifying the +field, for example `ZARR_ARRAY__ORDER=Q`. Programmatic `config.set()` validates +keys but continues to defer value validation to the setting's use site. + +The configuration can also be read from YAML files. Environment variables and +YAML files are read by [`donfig`](https://donfig.readthedocs.io/), so zarr uses +donfig's [standard search +locations](https://donfig.readthedocs.io/en/latest/configuration.html#yaml-files), +in increasing order of precedence: + +- `/etc/zarr/` (override this directory with the `ZARR_ROOT_CONFIG` + environment variable), +- `/etc/zarr/` and each entry in Python's `site.PREFIXES` (e.g. + inside a virtual environment), +- `~/.config/zarr/`, +- the path in the `ZARR_CONFIG` environment variable, which may point at a + single file or a directory and takes precedence over all of the above. + +Place a `zarr.yaml` in any of these directories, or point `ZARR_CONFIG` at a +specific file. YAML files contain configuration fields only: environment +controls such as `benchmark_clear_cache` are not valid YAML config keys. +Unrecognized keys are ignored with a warning during collection; recognized +values are checked when the typed configuration is created. Configuration options include the following: @@ -54,8 +85,5 @@ This is the current default configuration: ```python exec="true" session="config" source="above" result="ansi" from pprint import pprint -import io -output = io.StringIO() -zarr.config.pprint(stream=output, width=60) -print(output.getvalue()) +pprint(zarr.config.to_dict()) ``` diff --git a/pyproject.toml b/pyproject.toml index 4f5986edc7..3dcdce69c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -532,6 +532,39 @@ title_format = "## {version} ({project_date})" issue_format = "[#{issue}](https://github.com/zarr-developers/zarr-python/pull/{issue})" start_string = "\n" +# Fragment categories. These mirror towncrier's built-in defaults, with an +# added ``changed`` category for behavior changes that are neither new features +# nor bugfixes (e.g. an intentional change to existing, documented behavior). +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Misc" +showcontent = false + [tool.codespell] ignore-words-list = "astroid" diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index c0ebb31353..66effd3f4a 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -1,78 +1,842 @@ """ -The config module is responsible for managing the configuration of zarr and is based on the Donfig python library. -For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations -in the registry and then select them in the config. +Typed configuration for zarr. -Example: - An implementation of the bytes codec in a class ``your.module.NewBytesCodec`` requires the value of ``codecs.bytes`` - to be ``your.module.NewBytesCodec``. Donfig can be configured programmatically, by environment variables, or from - YAML files in standard locations. +The module exposes a single `config` object (a `ZarrConfigManager` instance) that +holds all runtime settings. Values can be read, overridden, and restored through a +simple string-key API: - ```python - from your.module import NewBytesCodec - from zarr.core.config import register_codec, config +- `config.get(key)` — read a dotted-key value (e.g. `config.get("async.concurrency")`). +- `config.set({key: value})` — permanent override; also usable as a context manager to + restore the previous state on exit. +- `config.reset()` — rebuild from defaults + environment. +- `config.refresh()` — alias for `reset`; called by the registry after env changes. +- `config.defaults` — nested dict of built-in default values. +- `config.enable_gpu()` — switch buffer/ndbuffer to GPU implementations. - register_codec("bytes", NewBytesCodec) - config.set({"codecs.bytes": "your.module.NewBytesCodec"}) - ``` +Environment variables use the `ZARR_` prefix and `__` for nesting: - Instead of setting the value programmatically with ``config.set``, you can also set the value with an environment - variable. The environment variable ``ZARR_CODECS__BYTES`` can be set to ``your.module.NewBytesCodec``. The double - underscore ``__`` is used to indicate nested access. +```bash +export ZARR_CODECS__BYTES="your.module.NewBytesCodec" +``` - ```bash - export ZARR_CODECS__BYTES="your.module.NewBytesCodec" - ``` +Programmatic override: -For more information, see the Donfig documentation at https://github.com/pytroll/donfig. +```python +from your.module import NewBytesCodec +from zarr.core.config import config + +config.set({"codecs.bytes": "your.module.NewBytesCodec"}) +``` + +For selecting custom implementations of codecs, pipelines, buffers, and ndbuffers, +register the implementation in the registry first, then set the path via `config.set`. """ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Literal, cast +import difflib +import os +import threading +import warnings +from collections.abc import Iterator, Mapping +from dataclasses import dataclass, field, fields, is_dataclass, replace +from types import MappingProxyType +from typing import Any, Literal, Self, TypedDict, cast, get_type_hints, overload -from donfig import Config as DConfig +from donfig.config_obj import canonical_name from zarr.core.json_parse import parse_field +from zarr.errors import ZarrDeprecationWarning, ZarrUserWarning -if TYPE_CHECKING: - from donfig.config_obj import ConfigSet +DEFAULT_CODECS: dict[str, str] = { + "blosc": "zarr.codecs.blosc.BloscCodec", + "gzip": "zarr.codecs.gzip.GzipCodec", + "zstd": "zarr.codecs.zstd.ZstdCodec", + "bytes": "zarr.codecs.bytes.BytesCodec", + "endian": "zarr.codecs.bytes.BytesCodec", + "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", + "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", + "transpose": "zarr.codecs.transpose.TransposeCodec", + "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", + "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", + "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", + "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", + "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", + "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", + "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", + "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", + "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", + "numcodecs.astype": "zarr.codecs.numcodecs.AsType", + "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", + "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", + "numcodecs.delta": "zarr.codecs.numcodecs.Delta", + "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", + "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", + "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", + "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", + "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", + "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", + "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", + "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", + "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", + "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", +} +# Map serialized dotted-key segments to Python field names where they differ +# (Python keywords cannot be used as identifiers). +_FIELD_ALIASES: dict[str, str] = {"async": "async_"} +_SERIALIZED_NAMES: dict[str, str] = {v: k for k, v in _FIELD_ALIASES.items()} + + +class _ConfigNode: + """Mixin giving the frozen config dataclasses donfig-style item access. + + donfig returned configuration subtrees as plain `dict`s, so callers could + write `config.get("array")["order"]`. The typed config returns dataclass + instances instead; this mixin restores subscripting (`node["order"]`, and + dotted `node["a.b"]`) alongside the typed attribute access (`node.order`), + raising `KeyError` for unknown keys just like the old dicts did. + + `__slots__ = ()` keeps the subclasses fully slotted (no `__dict__`). + """ + + __slots__ = () + + def __getitem__(self, key: str) -> object: + # `self` is always a config node (a frozen dataclass); `get_path` walks + # such nodes structurally, so the cast is sound. + return get_path(cast("ZarrConfig", self), key) + + def __contains__(self, key: object) -> bool: + # Mirror donfig's `"array" in config.get(...)` support. Only string keys + # can resolve; a non-string (or unknown key) is simply not contained. + if not isinstance(key, str): + return False + try: + get_path(cast("ZarrConfig", self), key) + except KeyError: + return False + return True + + def __iter__(self) -> Iterator[str]: + # Yield immediate child key names (serialized), so iteration, `keys()`, + # and `dict(node)` behave like the plain dicts donfig returned. Note this + # deliberately does NOT make the node a `collections.abc.Mapping`: the + # internal `isinstance(obj, Mapping)` checks must stay false for config + # nodes so they are distinguished from the open `codecs` mapping. + return iter(_children(self)) + + def keys(self) -> list[str]: + return _children(self) + + def __len__(self) -> int: + return len(_children(self)) + + +@dataclass(frozen=True, slots=True) +class ArraySettings(_ConfigNode): + order: Literal["C", "F"] = "C" + write_empty_chunks: bool = False + read_missing_chunks: bool = True + target_shard_size_bytes: int | None = None + rectilinear_chunks: bool = False + sharding_coalesce_max_gap_bytes: int = 1 << 20 + sharding_coalesce_max_bytes: int = 16 << 20 -class BadConfigError(ValueError): - _msg = "bad Config: %r" +@dataclass(frozen=True, slots=True) +class AsyncSettings(_ConfigNode): + concurrency: int = 10 + timeout: float | None = None -class Config(DConfig): # type: ignore[misc] - """The Config will collect configuration from config files and environment variables - Example environment variables: - Grabs environment variables of the form "ZARR_FOO__BAR_BAZ=123" and - turns these into config variables of the form ``{"foo": {"bar-baz": 123}}`` - It transforms the key and value in the following way: +@dataclass(frozen=True, slots=True) +class ThreadingSettings(_ConfigNode): + max_workers: int | None = None - - Lower-cases the key text - - Treats ``__`` (double-underscore) as nested access - - Calls ``ast.literal_eval`` on the value +@dataclass(frozen=True, slots=True) +class CodecPipelineSettings(_ConfigNode): + # FusedCodecPipeline remains opt-in via codec_pipeline.path. + path: str = "zarr.core.codec_pipeline.BatchedCodecPipeline" + batch_size: int = 1 + # Only read by FusedCodecPipeline; BatchedCodecPipeline ignores it. + max_workers: int | None = None + + +@dataclass(frozen=True, slots=True) +class ZarrConfig(_ConfigNode): + default_zarr_format: Literal[2, 3] = 3 + array: ArraySettings = field(default_factory=ArraySettings) + async_: AsyncSettings = field(default_factory=AsyncSettings) + threading: ThreadingSettings = field(default_factory=ThreadingSettings) + json_indent: int = 2 + codec_pipeline: CodecPipelineSettings = field(default_factory=CodecPipelineSettings) + # A plain dict (not MappingProxyType) so snapshots stay picklable / + # deep-copyable; public read access goes through the manager's `codecs` + # property, which wraps this in a read-only view. + codecs: Mapping[str, str] = field(default_factory=lambda: dict(DEFAULT_CODECS)) + buffer: str = "zarr.buffer.cpu.Buffer" + ndbuffer: str = "zarr.buffer.cpu.NDBuffer" + + +def make_default_config() -> ZarrConfig: + """Return a fresh `ZarrConfig` populated with the built-in defaults.""" + return ZarrConfig() + + +def _resolve_field(obj: object, segment: str) -> str: + """Translate a serialized key segment to the dataclass field name.""" + field_name = _FIELD_ALIASES.get(segment, segment) + return cast(str, canonical_name(field_name, dict.fromkeys(_children(obj)))) + + +def get_path(cfg: ZarrConfig, key: str) -> object: + """Read a dotted-string key from a `ZarrConfig` snapshot. + + Raises + ------ + KeyError + If the key does not resolve to a value. """ + obj: object = cfg + segments = key.split(".") + for i, segment in enumerate(segments): + if isinstance(obj, Mapping): + # remaining segments index into an open mapping (e.g. codecs.*) + remainder = ".".join(segments[i:]) + try: + return obj[canonical_name(remainder, obj)] + except KeyError: + raise KeyError(key) from None + # A prior segment resolved to a scalar leaf, but the key has more + # segments — descend no further. Without this guard, `hasattr` would + # match ordinary Python attributes/methods (e.g. `array.order.upper` + # returning `str.upper`, or `default_zarr_format.numerator` returning + # the int's numerator) instead of raising for the invalid key. + if not is_dataclass(obj): + raise KeyError(key) + field_name = _resolve_field(obj, segment) + if field_name not in {f.name for f in fields(obj)}: + raise KeyError(key) + obj = getattr(obj, field_name) + return obj - def reset(self) -> None: - self.clear() - self.refresh() - def enable_gpu(self) -> ConfigSet: - """ - Configure Zarr to use GPUs where possible. +def replace_path(cfg: ZarrConfig, key: str, value: object) -> ZarrConfig: + """Return a new `ZarrConfig` with the dotted-string key set to ``value``.""" + segments = key.split(".") + return cast(ZarrConfig, _replace_recursive(cfg, segments, value, key)) + + +# `obj: Any` is load-bearing here: the function dispatches dynamically between a +# `Mapping` (codecs subtree) and a dataclass instance, and `dataclasses.replace` +# requires a dataclass-typed argument that `object` would reject. +def _replace_recursive(obj: Any, segments: list[str], value: object, key: str) -> object: + segment = segments[0] + if isinstance(obj, Mapping): + remainder = ".".join(segments) + # Plain dict (see the `codecs` field note); the manager property wraps + # it read-only for public access. + return {**obj, canonical_name(remainder, obj): value} + if not is_dataclass(obj): + # `key` tries to descend past a scalar leaf (e.g. `array.order.upper`). + raise KeyError(key) + field_name = _resolve_field(obj, segment) + if field_name not in {f.name for f in fields(obj)}: + raise KeyError(key) + # `is_dataclass` narrows `obj` to `... | type[...]`, which `replace` rejects; + # at runtime `obj` is always a dataclass *instance* here, so re-widen to Any. + node: Any = obj + child = getattr(node, field_name) + if len(segments) == 1: + # Refuse to overwrite a structured subtree (a nested dataclass) wholesale + # — doing so would drop its sibling fields and break typed attribute + # access. Set leaf keys instead. The open `codecs` mapping is not a + # dataclass, so wholesale replacement there is still allowed. + if is_dataclass(child): + raise TypeError( + f"Cannot assign to the structured config subtree {key!r} directly; " + f"set leaf keys instead, e.g. config.set({{'{key}.': ...}})." + ) + return replace(node, **{field_name: value}) + new_child = _replace_recursive(child, segments[1:], value, key) + return replace(node, **{field_name: new_child}) + + +def delete_path(cfg: ZarrConfig, key: str) -> ZarrConfig: + """Return a new `ZarrConfig` with ``key`` removed from an open mapping. + + Only keys inside an open mapping (``codecs.*``) can be deleted; structured + fields always exist and cannot be removed. Used to undo a scoped ``set`` that + *added* a new codec key when its ``with`` block exits. + """ + segments = key.split(".") + return cast("ZarrConfig", _delete_recursive(cfg, segments, key)) + + +def _delete_recursive(obj: Any, segments: list[str], key: str) -> object: + if isinstance(obj, Mapping): + remainder = ".".join(segments) + remainder = canonical_name(remainder, obj) + return {k: v for k, v in obj.items() if k != remainder} + if not is_dataclass(obj): + raise KeyError(key) + field_name = _resolve_field(obj, segments[0]) + if field_name not in {f.name for f in fields(obj)}: + raise KeyError(key) + node: Any = obj + child = getattr(node, field_name) + if len(segments) == 1: + # a structured field can't be deleted; only open-mapping leaves can + raise KeyError(key) + return replace(node, **{field_name: _delete_recursive(child, segments[1:], key)}) + + +_ROSTER_LIMIT = 10 + + +def _children(obj: object) -> list[str]: + """Return the immediate child key names of a config node (else an empty list).""" + if isinstance(obj, Mapping): + return list(obj) + if is_dataclass(obj): + return [_SERIALIZED_NAMES.get(f.name, f.name) for f in fields(obj)] + return [] + + +def _resolve_for_suggestion(cfg: ZarrConfig, key: str) -> tuple[str, list[str], str]: + """Walk ``key`` as far as it resolves. + + Returns the deepest resolvable dotted prefix, that node's child key names, + and the first segment that failed to resolve (the remainder is treated as a + single key once an open mapping like ``codecs`` is reached). For + ``"array.bogus"`` this is ``("array", [], "bogus")``; + for an unknown top-level key, ``("", [], )``. + """ + obj: object = cfg + prefix = "" + segments = key.split(".") + for i, segment in enumerate(segments): + if isinstance(obj, Mapping): + # the remainder indexes into an open mapping as a single key + return prefix, _children(obj), ".".join(segments[i:]) + if not is_dataclass(obj): + # a prior segment resolved to a scalar leaf; nothing deeper is valid + return prefix, _children(obj), segment + field_name = _resolve_field(obj, segment) + if field_name not in {f.name for f in fields(obj)}: + return prefix, _children(obj), segment + obj = getattr(obj, field_name) + prefix = f"{prefix}.{segment}" if prefix else segment + return prefix, _children(obj), "" + + +def _unknown_key_error(key: str, cfg: ZarrConfig) -> KeyError: + """Build a `KeyError` for an unknown config key. + + Resolves ``key`` to the deepest valid level, then suggests the closest child + key there if one is similar enough; otherwise lists the available keys at + that level (capped at `_ROSTER_LIMIT`). + """ + msg = f"{key!r} is not a valid configuration key." + prefix, children, failed = _resolve_for_suggestion(cfg, key) + matches = difflib.get_close_matches(failed, children, n=1) if failed != "" else [] + if len(matches) > 0: + suggestion = f"{prefix}.{matches[0]}" if prefix != "" else matches[0] + return KeyError(f"{msg} Did you mean {suggestion!r}?") + if len(children) > 0: + shown = sorted(children) + roster = ", ".join(shown[:_ROSTER_LIMIT]) + if len(shown) > _ROSTER_LIMIT: + roster += f", ... ({len(shown) - _ROSTER_LIMIT} more)" + where = f" under {prefix!r}" if prefix != "" else "" + msg = f"{msg} Valid keys{where}: {roster}." + return KeyError(msg) + + +def to_nested_dict(cfg: ZarrConfig) -> dict[str, Any]: + """Convert a `ZarrConfig` to a donfig-style nested dict (serialized keys). + + Returns a heterogeneous, JSON-like tree (nested dicts and scalars) that + callers navigate by key, so `Any` values are appropriate here. + """ + + # `obj: Any` is also load-bearing: `dataclasses.fields` requires a + # dataclass-typed argument that `object` would reject. + def convert(obj: Any) -> Any: + if isinstance(obj, Mapping): + return dict(obj) + if hasattr(type(obj), "__dataclass_fields__"): + out: dict[str, Any] = {} + for f in fields(obj): + serialized = _SERIALIZED_NAMES.get(f.name, f.name) + out[serialized] = convert(getattr(obj, f.name)) + return out + return obj + + return convert(cfg) # type: ignore[no-any-return] + + +def _config_value_type(cfg: ZarrConfig, key: str) -> Any: + """Resolve a schema type, including names in the open codecs mapping.""" + node: object = cfg + segments = key.split(".") + for index, segment in enumerate(segments): + if isinstance(node, Mapping): + if not ".".join(segments[index:]): + raise KeyError(key) + return str + if not is_dataclass(node): + raise KeyError(key) + field_name = _resolve_field(node, segment) + hints = get_type_hints(type(node)) + if field_name not in hints: + raise KeyError(key) + value_type = hints[field_name] + node = getattr(node, field_name) + return value_type + + +def apply_overrides(cfg: ZarrConfig, overrides: Mapping[str, object]) -> ZarrConfig: + """Validate and apply collected leaf values; unknown keys are errors here.""" + for key, value in overrides.items(): + value_type = _config_value_type(cfg, key) + cfg = replace_path(cfg, key, parse_field(value, value_type, key)) + return cfg + + +# These names belong to other Zarr consumers, not the runtime config schema. +# Donfig reads discovery controls directly from os.environ; benchmarks read +# their cache-control flag there as well. Collection never mutates os.environ. +_ENVIRONMENT_CONTROLS = frozenset({"ZARR_CONFIG", "ZARR_ROOT_CONFIG", "ZARR_BENCHMARK_CLEAR_CACHE"}) + + +class CollectedEnvironment(TypedDict): + config: dict[str, str] + controls: dict[str, str] + + +def collect_environment(environment: Mapping[str, str]) -> CollectedEnvironment: + """Classify recognized ZARR_ names and warn about unknown environment names. + + Config names are derived from the schema, with an open ``codecs.*`` + namespace. Control values remain raw and are left to their owning consumer. + Parsing and value validation happen after this stage. + """ + result: CollectedEnvironment = {"config": {}, "controls": {}} + schema = make_default_config() + for name, value in environment.items(): + if not name.startswith("ZARR_"): + continue + if name in _ENVIRONMENT_CONTROLS: + result["controls"][name] = value + continue + key = name[len("ZARR_") :].lower().replace("__", ".") + try: + _config_value_type(schema, key) + except KeyError: + warnings.warn( + f"Unrecognized zarr environment variable {name!r} (config key {key!r}) — ignoring.", + ZarrUserWarning, + stacklevel=2, + ) + else: + result["config"][name] = value + return result + + +def collect_config() -> dict[str, object]: + """Collect known config keys through donfig, preserving parsing and precedence. + + Environment controls are recognized separately and never become config + values. Unknown keys in external input are warned about and skipped here; + recognized values are left for ``create_config`` to validate. + """ + import donfig + + defaults = make_default_config() + environment = collect_environment(os.environ) + # Donfig merges external sources; construction supplies typed defaults. + # Keeping defaults out also lets invalid leaf values reach our validator + # instead of failing inside donfig's recursive merge against a scalar. + reader = donfig.Config("zarr", env=environment["config"]) + return _collect_config_values(defaults, reader.config) + + +def _collect_config_values( + schema: ZarrConfig, data: Mapping[str, object], prefix: str = "" +) -> dict[str, object]: + """Flatten recognized namespaces, preserving malformed leaves for validation.""" + collected: dict[str, object] = {} + for name, value in data.items(): + key = f"{prefix}.{name}" if prefix else name + try: + value_type = _config_value_type(schema, key) + except KeyError: + warnings.warn( + f"Unrecognized zarr config key {key!r} from environment or YAML — ignoring.", + ZarrUserWarning, + stacklevel=2, + ) + else: + if isinstance(value, Mapping) and ( + is_dataclass(value_type) or key == "codecs" or key.startswith("codecs.") + ): + collected.update(_collect_config_values(schema, value, key)) + else: + collected[key] = value + return collected + + +def create_config(overrides: Mapping[str, object]) -> ZarrConfig: + """Create a typed config from collected values, without reading external state.""" + return apply_overrides(make_default_config(), overrides) + + +def build_config() -> ZarrConfig: + """Collect external configuration, then validate and construct the snapshot.""" + return create_config(collect_config()) + + +_MISSING = object() + + +# A per-key record of how to undo a scoped `set`: (resolved_key, existed_before, +# previous_value). If the key did not exist before (a newly added `codecs.*` +# entry), it is removed on exit instead of restored. +_RestoreEntry = tuple[str, bool, object] + + +class _ConfigSet: + """Context manager returned by ``ZarrConfigManager.set``. + + ``set`` applies its overrides immediately to the process-global config, so a + bare ``config.set(...)`` is permanent and visible from every thread (donfig's + last-writer-wins semantics). Used as a ``with`` block, the overrides are + reverted on ``__exit__``: each key it set is restored to its previous value + (or removed, if it was newly added), matching ``donfig``. Reverting only the + keys it touched leaves concurrent changes to *other* keys intact. + + Note: like ``donfig``, a ``with config.set(...)`` is **not** isolated to the + calling thread / async task — during the block the change is globally visible. + Context-local isolation is a possible future enhancement (see gh-4101). + """ + + def __init__(self, manager: ZarrConfigManager, restore: list[_RestoreEntry]) -> None: + self._manager = manager + self._restore = restore + + def __enter__(self) -> Self: + # The overrides were already applied globally by `set`; nothing to do. + return self + + def __exit__(self, *exc: object) -> None: + self._manager._revert(self._restore) + + +class ZarrConfigManager: + """Typed, donfig-compatible configuration object.""" + + def __init__(self) -> None: + self._base: ZarrConfig = build_config() + # Serializes read-modify-write of the process-global `_base` so + # concurrent `set`s / reverts to different keys don't lose updates + # (each rebuilds a whole immutable snapshot from `_base`). + self._lock = threading.Lock() + + def __getstate__(self) -> dict[str, Any]: + # Locks cannot be copied or pickled. Capture state under the lock, then + # let the copy/pickle protocol process the snapshot independently. + with self._lock: + return {key: value for key, value in self.__dict__.items() if key != "_lock"} + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + self._lock = threading.Lock() + + # --- state resolution ------------------------------------------------- + def _current(self) -> ZarrConfig: + # A single process-global snapshot: a `set` (bare or scoped) is globally + # visible, and a `with config.set(...)` reverts its keys on exit. This + # mirrors donfig; there is no per-thread/async overlay. + return self._base + + def _revert(self, restore: list[_RestoreEntry]) -> None: + # Undo a scoped `set` on `with`-block exit: restore each key it touched to + # its previous value (or remove a key it newly added), leaving concurrent + # changes to other keys intact. + with self._lock: + cfg = self._base + for resolved, existed, old in reversed(restore): + cfg = replace_path(cfg, resolved, old) if existed else delete_path(cfg, resolved) + self._base = cfg + + # --- typed attribute access ------------------------------------------ + @property + def default_zarr_format(self) -> Literal[2, 3]: + return self._current().default_zarr_format + + @property + def array(self) -> ArraySettings: + return self._current().array + + @property + def async_(self) -> AsyncSettings: + return self._current().async_ + + @property + def threading(self) -> ThreadingSettings: + return self._current().threading + + @property + def codec_pipeline(self) -> CodecPipelineSettings: + return self._current().codec_pipeline + + @property + def json_indent(self) -> int: + return self._current().json_indent + + @property + def codecs(self) -> Mapping[str, str]: + # Read-only view: the underlying snapshot field is a plain dict (kept + # picklable), but callers must not mutate a live snapshot in place. + return MappingProxyType(self._current().codecs) + + @property + def buffer(self) -> str: + return self._current().buffer + + @property + def ndbuffer(self) -> str: + return self._current().ndbuffer + + # --- string API: get -------------------------------------------------- + @overload + def get(self, key: Literal["default_zarr_format"]) -> Literal[2, 3]: ... + @overload + def get(self, key: Literal["array.order"]) -> Literal["C", "F"]: ... + @overload + def get(self, key: Literal["array.write_empty_chunks"]) -> bool: ... + @overload + def get(self, key: Literal["array.read_missing_chunks"]) -> bool: ... + @overload + def get(self, key: Literal["array.target_shard_size_bytes"]) -> int | None: ... + @overload + def get(self, key: Literal["array.rectilinear_chunks"]) -> bool: ... + @overload + def get(self, key: Literal["array.sharding_coalesce_max_gap_bytes"]) -> int: ... + @overload + def get(self, key: Literal["array.sharding_coalesce_max_bytes"]) -> int: ... + @overload + def get(self, key: Literal["async.concurrency"]) -> int: ... + @overload + def get(self, key: Literal["async.timeout"]) -> float | None: ... + @overload + def get(self, key: Literal["threading.max_workers"]) -> int | None: ... + @overload + def get(self, key: Literal["json_indent"]) -> int: ... + @overload + def get(self, key: Literal["codec_pipeline.path"]) -> str: ... + @overload + def get(self, key: Literal["codec_pipeline.batch_size"]) -> int: ... + @overload + def get(self, key: Literal["codec_pipeline.max_workers"]) -> int | None: ... + @overload + def get(self, key: Literal["buffer"]) -> str: ... + @overload + def get(self, key: Literal["ndbuffer"]) -> str: ... + @overload + # The fallback `-> Any` is deliberate: it lets `config.get("codecs", {})` be + # used as a mapping (e.g. `.get(name)` in the registry) and supports unknown + # keys. `object` here would force every such call site to narrow first. + def get(self, key: str, default: object = ...) -> Any: ... + + def get(self, key: str, default: object = _MISSING) -> Any: + resolved = self._apply_deprecation(key, raise_on_removed=False) + if resolved is None: + # Key was removed; treat as absent — honour the caller's default. + if default is _MISSING: + raise KeyError(key) + return default + current = self._current() + try: + return get_path(current, resolved) + except KeyError: + if default is _MISSING: + raise _unknown_key_error(key, current) from None + return default + + # --- string API: set -------------------------------------------------- + # + # NOTE: `set` accepts `Mapping[str, Any]`, so — unlike `get`, which is fully + # typed via per-key overloads — it does NOT statically validate values: + # `config.set({"array.order": "Q"})` is not a type error; invalid values + # surface at use sites. External values are validated by `create_config`. + # + # Static value typing would require an *open* TypedDict — declared structured + # keys validated by type, PLUS arbitrary `codecs.` string keys allowed + # (PEP 728 `extra_items`/`closed`). mypy (2.x) supports PEP 728 in no syntax + # and offers no feature flag for it. A *closed* TypedDict would instead reject + # the open codec-selection idiom + # `config.set({"codecs.bytes": "your.module.NewBytesCodec"})` and any + # dynamically built `dict[str, Any]` — a backwards-compatibility regression + # (the `codecs` namespace maps a codec name to a class path and is extended at + # runtime by users/plugins, so its keys cannot be enumerated statically). + # So `set` is intentionally permissive and validated at runtime: unknown + # structured keys raise (see `replace_path`), while `codecs.*` stays writable. + # + # REVISIT when mypy ships PEP 728 open-TypedDict support, or if zarr adopts a + # type checker that supports it (e.g. pyright's open/closed TypedDicts). At + # that point `set` can take an open TypedDict for static value validation + # while keeping `codecs.*` open. + def set(self, updates: Mapping[str, object] | None = None, **kwargs: object) -> _ConfigSet: + """Apply one or more config overrides. + + Accepts either a mapping of dotted keys to values, keyword arguments + (for top-level keys), or both:: + + config.set({"array.order": "F"}) + config.set(default_zarr_format=2) + + `set` validates *keys* — an unknown key raises with a suggestion — but + does **not** validate *values*: `config.set({"array.order": "Q"})` is + accepted, and the invalid value surfaces later at its use site rather + than here. Static value typing is prevented by the open `codecs.*` + namespace (see the implementation comment above). Values loaded from + environment variables and YAML are validated by `create_config`. """ + all_updates: dict[str, object] = {} + if updates: + all_updates.update(updates) + all_updates.update((key.replace("__", "."), value) for key, value in kwargs.items()) + # Apply immediately and globally (donfig semantics). Hold the lock across + # the read-modify-write of `_base` so concurrent `set`s to different keys + # don't clobber each other (each rebuilds a full snapshot). Record how to + # undo each key so a `with config.set(...)` can revert on exit. + restore: list[_RestoreEntry] = [] + with self._lock: + cfg = self._base + for key, value in all_updates.items(): + resolved = self._apply_deprecation(key, raise_on_removed=True) + try: + old = get_path(cfg, resolved) + existed = True + except KeyError: + old = None + existed = False + try: + cfg = replace_path(cfg, resolved, value) + except KeyError: + raise _unknown_key_error(key, cfg) from None + restore.append((resolved, existed, old)) + self._base = cfg + return _ConfigSet(self, restore) + + # --- lifecycle -------------------------------------------------------- + def reset(self) -> None: + # Rebuild the global base from defaults + env/YAML. Build outside the lock + # (it reads env/YAML) and swap atomically under it. + new_base = build_config() + with self._lock: + self._base = new_base + + def refresh(self) -> None: + self.reset() + + def enable_gpu(self) -> _ConfigSet: return self.set( {"buffer": "zarr.buffer.gpu.Buffer", "ndbuffer": "zarr.buffer.gpu.NDBuffer"} ) + # --- compat / introspection ------------------------------------------ + def __getitem__(self, key: str) -> Any: + # donfig-style item access: `config["array.order"]` mirrors `get`. + return self.get(key) + + def __contains__(self, key: object) -> bool: + # donfig-style membership: `"array.order" in config`. + if not isinstance(key, str): + return False + try: + self.get(key) + except KeyError: + return False + return True + + def __iter__(self) -> Iterator[str]: + # Defining `__getitem__` above would otherwise make Python fall back to + # the legacy integer-index iteration protocol (`config[0]` -> confusing + # `'int' object has no attribute 'split'`). The manager is a keyed config, + # not a sequence, so fail clearly instead. + raise TypeError( + "ZarrConfigManager is not iterable; use config.to_dict() to iterate " + "its contents, or config.get() for a single value." + ) + + @property + def defaults(self) -> dict[str, Any]: + return to_nested_dict(make_default_config()) + + def to_dict(self) -> dict[str, Any]: + return to_nested_dict(self._current()) + + def update(self, updates: Mapping[str, object]) -> None: + self.set(updates) + + def pprint(self) -> None: + import pprint as _pp + + _pp.pprint(self.to_dict()) + + # --- deprecations ----------------------------------------------------- + @overload + def _apply_deprecation(self, key: str, *, raise_on_removed: Literal[True]) -> str: ... + @overload + def _apply_deprecation(self, key: str, *, raise_on_removed: Literal[False]) -> str | None: ... + + def _apply_deprecation(self, key: str, *, raise_on_removed: bool) -> str | None: + """Resolve a possibly-deprecated config key. + + Parameters + ---------- + key : str + The dotted config key supplied by the caller. + raise_on_removed : bool + When `True` (used by `set`), raise `BadConfigError` if the key has been + removed. When `False` (used by `get`), return `None` instead so the + caller can treat the key as absent and honour the caller's default. + + Returns + ------- + str or None + The canonical (possibly redirected) key, or `None` when the key was + removed and `raise_on_removed` is `False`. + """ + if key not in deprecations: + return key + new_key = deprecations[key] + if new_key is None: + if raise_on_removed: + raise BadConfigError( + f"Configuration key {key!r} has been removed and no longer has any effect." + ) + return None + warnings.warn( + f"Configuration key {key!r} has been renamed to {new_key!r}.", + ZarrDeprecationWarning, + stacklevel=3, + ) + return new_key + + +class BadConfigError(ValueError): + _msg = "bad Config: %r" + # these keys were removed from the config as part of the 3.1.0 release. -# these deprecations should be removed in 3.1.1 or thereabouts. -deprecations = { +# These deprecations should be removed in 3.1.1 or thereabouts. +deprecations: dict[str, str | None] = { "array.v2_default_compressor.numeric": None, "array.v2_default_compressor.string": None, "array.v2_default_compressor.bytes": None, @@ -89,76 +853,13 @@ def enable_gpu(self) -> ConfigSet: "array.v3_default_compressors": None, } -# The default configuration for zarr -config = Config( - "zarr", - defaults=[ - { - "default_zarr_format": 3, - "array": { - "order": "C", - "write_empty_chunks": False, - "read_missing_chunks": True, - "target_shard_size_bytes": None, - "rectilinear_chunks": False, - "sharding_coalesce_max_gap_bytes": 1 << 20, # 1 MiB - "sharding_coalesce_max_bytes": 16 << 20, # 16 MiB - }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, - "json_indent": 2, - "codec_pipeline": { - # FusedCodecPipeline is the faster synchronous pipeline, but it stays - # opt-in for now so behavior is unchanged for existing users. Early - # adopters can switch with - # zarr.config.set( - # {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} - # ) - "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", - "batch_size": 1, - # Only read by FusedCodecPipeline (BatchedCodecPipeline ignores it). - "max_workers": None, - }, - "codecs": { - "blosc": "zarr.codecs.blosc.BloscCodec", - "gzip": "zarr.codecs.gzip.GzipCodec", - "zstd": "zarr.codecs.zstd.ZstdCodec", - "bytes": "zarr.codecs.bytes.BytesCodec", - "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 - "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", - "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", - "transpose": "zarr.codecs.transpose.TransposeCodec", - "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", - "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", - "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", - "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", - "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", - "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", - "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", - "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", - "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", - "numcodecs.astype": "zarr.codecs.numcodecs.AsType", - "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", - "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", - "numcodecs.delta": "zarr.codecs.numcodecs.Delta", - "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", - "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", - "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", - "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", - "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", - "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", - "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", - "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", - "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", - "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", - }, - "buffer": "zarr.buffer.cpu.Buffer", - "ndbuffer": "zarr.buffer.cpu.NDBuffer", - } - ], - deprecations=deprecations, -) - - -def parse_indexing_order(data: Any) -> Literal["C", "F"]: +# Backwards-compatible alias: before the typed rewrite, this module exposed the +# donfig subclass as `Config`. Keep the name so `from zarr.core.config import +# Config` and `isinstance(x, Config)` continue to work for downstream code. +Config = ZarrConfigManager + +config = ZarrConfigManager() + + +def parse_indexing_order(data: object) -> Literal["C", "F"]: return cast("Literal['C', 'F']", parse_field(data, Literal["C", "F"], "order")) diff --git a/tests/benchmarks/test_config.py b/tests/benchmarks/test_config.py new file mode 100644 index 0000000000..e08c70c665 --- /dev/null +++ b/tests/benchmarks/test_config.py @@ -0,0 +1,20 @@ +"""Benchmark controls are recognized during collection; typos still warn.""" + +import pytest + +from zarr.core.config import ZarrConfigManager +from zarr.errors import ZarrUserWarning + + +def test_cache_control_flag_allows_config_reset(monkeypatch: pytest.MonkeyPatch) -> None: + cfg = ZarrConfigManager() + before = cfg.to_dict() + monkeypatch.setenv("ZARR_BENCHMARK_CLEAR_CACHE", "1") + cfg.reset() + assert cfg.to_dict() == before + + +def test_unknown_benchmark_config_key_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ZARR_BENCHMARK_CLEAR_CACH", "1") + with pytest.raises(ZarrUserWarning, match="'benchmark_clear_cach'"): + ZarrConfigManager() diff --git a/tests/benchmarks/test_e2e.py b/tests/benchmarks/test_e2e.py index 9720778d8f..f0d34de77a 100644 --- a/tests/benchmarks/test_e2e.py +++ b/tests/benchmarks/test_e2e.py @@ -88,7 +88,7 @@ def _data(shape: tuple[int]) -> np.ndarray: Layout(shape=(1_000_000_000,), chunks=(100_000,), shards=(100_000 * 100,)), ) -_PIPELINE_SETTINGS = { +_PIPELINE_SETTINGS: dict[str, dict[str, str | int | None]] = { "batched": {"codec_pipeline.path": "zarr.core.codec_pipeline.BatchedCodecPipeline"}, "fused_full_threaded": { "codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline", diff --git a/tests/test_config.py b/tests/test_config.py index c22aac7603..b01feb0357 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -22,7 +22,7 @@ from zarr.core.buffer import NDBuffer from zarr.core.buffer.core import Buffer from zarr.core.codec_pipeline import BatchedCodecPipeline -from zarr.core.config import BadConfigError, config +from zarr.core.config import DEFAULT_CODECS, BadConfigError, config from zarr.core.indexing import SelectorTuple from zarr.errors import ChunkNotFoundError, UnknownCodecError, ZarrUserWarning from zarr.registry import ( @@ -45,67 +45,29 @@ def test_config_defaults_set() -> None: - # regression test for available defaults - assert ( - config.defaults - == [ - { - "default_zarr_format": 3, - "array": { - "order": "C", - "write_empty_chunks": False, - "read_missing_chunks": True, - "target_shard_size_bytes": None, - "rectilinear_chunks": False, - "sharding_coalesce_max_gap_bytes": 1 << 20, - "sharding_coalesce_max_bytes": 16 << 20, - }, - "async": {"concurrency": 10, "timeout": None}, - "threading": {"max_workers": None}, - "json_indent": 2, - "codec_pipeline": { - "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", - "batch_size": 1, - "max_workers": None, - }, - "codecs": { - "blosc": "zarr.codecs.blosc.BloscCodec", - "gzip": "zarr.codecs.gzip.GzipCodec", - "zstd": "zarr.codecs.zstd.ZstdCodec", - "bytes": "zarr.codecs.bytes.BytesCodec", - "endian": "zarr.codecs.bytes.BytesCodec", # compatibility with earlier versions of ZEP1 - "crc32c": "zarr.codecs.crc32c_.Crc32cCodec", - "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", - "transpose": "zarr.codecs.transpose.TransposeCodec", - "vlen-utf8": "zarr.codecs.vlen_utf8.VLenUTF8Codec", - "vlen-bytes": "zarr.codecs.vlen_utf8.VLenBytesCodec", - "numcodecs.bz2": "zarr.codecs.numcodecs.BZ2", - "numcodecs.crc32": "zarr.codecs.numcodecs.CRC32", - "numcodecs.crc32c": "zarr.codecs.numcodecs.CRC32C", - "numcodecs.lz4": "zarr.codecs.numcodecs.LZ4", - "numcodecs.lzma": "zarr.codecs.numcodecs.LZMA", - "numcodecs.zfpy": "zarr.codecs.numcodecs.ZFPY", - "numcodecs.adler32": "zarr.codecs.numcodecs.Adler32", - "numcodecs.astype": "zarr.codecs.numcodecs.AsType", - "numcodecs.bitround": "zarr.codecs.numcodecs.BitRound", - "numcodecs.blosc": "zarr.codecs.numcodecs.Blosc", - "numcodecs.delta": "zarr.codecs.numcodecs.Delta", - "numcodecs.fixedscaleoffset": "zarr.codecs.numcodecs.FixedScaleOffset", - "numcodecs.fletcher32": "zarr.codecs.numcodecs.Fletcher32", - "numcodecs.gzip": "zarr.codecs.numcodecs.GZip", - "numcodecs.jenkins_lookup3": "zarr.codecs.numcodecs.JenkinsLookup3", - "numcodecs.pcodec": "zarr.codecs.numcodecs.PCodec", - "numcodecs.packbits": "zarr.codecs.numcodecs.PackBits", - "numcodecs.shuffle": "zarr.codecs.numcodecs.Shuffle", - "numcodecs.quantize": "zarr.codecs.numcodecs.Quantize", - "numcodecs.zlib": "zarr.codecs.numcodecs.Zlib", - "numcodecs.zstd": "zarr.codecs.numcodecs.Zstd", - }, - "buffer": "zarr.buffer.cpu.Buffer", - "ndbuffer": "zarr.buffer.cpu.NDBuffer", - } - ] - ) + assert config.defaults == { + "default_zarr_format": 3, + "array": { + "order": "C", + "write_empty_chunks": False, + "read_missing_chunks": True, + "target_shard_size_bytes": None, + "rectilinear_chunks": False, + "sharding_coalesce_max_gap_bytes": 1 << 20, + "sharding_coalesce_max_bytes": 16 << 20, + }, + "async": {"concurrency": 10, "timeout": None}, + "threading": {"max_workers": None}, + "json_indent": 2, + "codec_pipeline": { + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", + "batch_size": 1, + "max_workers": None, + }, + "codecs": dict(DEFAULT_CODECS), + "buffer": "zarr.buffer.cpu.Buffer", + "ndbuffer": "zarr.buffer.cpu.NDBuffer", + } assert config.get("array.order") == "C" assert config.get("async.concurrency") == 10 assert config.get("async.timeout") is None @@ -207,7 +169,7 @@ class MockEnvCodecPipeline(CodecPipeline): @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) def test_config_codec_implementation(store: Store) -> None: # has default value - assert fully_qualified_name(get_codec_class("blosc")) == config.defaults[0]["codecs"]["blosc"] + assert fully_qualified_name(get_codec_class("blosc")) == config.defaults["codecs"]["blosc"] _mock = Mock() @@ -269,7 +231,7 @@ def test_config_ndbuffer_implementation(store: Store) -> None: def test_config_buffer_implementation() -> None: # has default value - assert config.defaults[0]["buffer"] == "zarr.buffer.cpu.Buffer" + assert config.defaults["buffer"] == "zarr.buffer.cpu.Buffer" arr = zeros(shape=(100,), store=StoreExpectingTestBuffer()) diff --git a/tests/test_config_collection.py b/tests/test_config_collection.py new file mode 100644 index 0000000000..e007e6eb81 --- /dev/null +++ b/tests/test_config_collection.py @@ -0,0 +1,151 @@ +"""Environment ownership and the boundary between collection and construction.""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING + +import pytest + +from zarr.core.config import collect_config, collect_environment, create_config +from zarr.errors import ZarrUserWarning + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.mark.parametrize( + ("environment", "config", "controls"), + [ + ({}, {}, {}), + ({"PATH": "/bin"}, {}, {}), + ({"ZARR_ARRAY__ORDER": "F"}, {"ZARR_ARRAY__ORDER": "F"}, {}), + ({"ZARR_CODEC-PIPELINE__MAX-WORKERS": "4"}, {"ZARR_CODEC-PIPELINE__MAX-WORKERS": "4"}, {}), + ( + {"ZARR_CODECS__CUSTOM_CODEC": "pkg.Custom"}, + {"ZARR_CODECS__CUSTOM_CODEC": "pkg.Custom"}, + {}, + ), + ( + { + "ZARR_CONFIG": "/tmp/zarr.yaml", + "ZARR_ROOT_CONFIG": "/tmp/config", + "ZARR_BENCHMARK_CLEAR_CACHE": "1", + "ZARR_ASYNC__CONCURRENCY": "8", + }, + {"ZARR_ASYNC__CONCURRENCY": "8"}, + { + "ZARR_CONFIG": "/tmp/zarr.yaml", + "ZARR_ROOT_CONFIG": "/tmp/config", + "ZARR_BENCHMARK_CLEAR_CACHE": "1", + }, + ), + ], +) +def test_collect_environment( + environment: dict[str, str], config: dict[str, str], controls: dict[str, str] +) -> None: + before = environment.copy() + with warnings.catch_warnings(): + warnings.simplefilter("error", ZarrUserWarning) + assert collect_environment(environment) == {"config": config, "controls": controls} + assert environment == before + + +@pytest.mark.parametrize( + "name", + [ + "ZARR_BENCHMARK_CLEAR_CACH", + "ZARR_ARRAY__ORDRE", + "ZARR_ARRAY__ORDER__UPPER", + "ZARR_FUTURE__KEY", + ], +) +def test_collect_environment_unknown_name_warns(name: str) -> None: + with pytest.warns(ZarrUserWarning, match=name) as caught: + result = collect_environment({name: "1", "ZARR_ARRAY__ORDER": "F"}) + assert len(caught) == 1 + assert result == {"config": {"ZARR_ARRAY__ORDER": "F"}, "controls": {}} + + +def test_collect_config_separates_controls_and_defers_value_validation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config_file = tmp_path / "zarr.yaml" + config_file.write_text("array:\n order: Q\n") + monkeypatch.setenv("ZARR_CONFIG", str(config_file)) + monkeypatch.setenv("ZARR_BENCHMARK_CLEAR_CACHE", "1") + with warnings.catch_warnings(): + warnings.simplefilter("error", ZarrUserWarning) + collected = collect_config() + assert collected["array.order"] == "Q" + assert "config" not in collected + assert "benchmark_clear_cache" not in collected + with pytest.raises(ValueError, match="array.order"): + create_config(collected) + + +def test_collect_config_preserves_invalid_leaf_mapping( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config_file = tmp_path / "zarr.yaml" + config_file.write_text("async:\n concurrency:\n invalid: 1\n") + monkeypatch.setenv("ZARR_CONFIG", str(config_file)) + with warnings.catch_warnings(): + warnings.simplefilter("error", ZarrUserWarning) + collected = collect_config() + assert collected["async.concurrency"] == {"invalid": 1} + with pytest.raises(ValueError, match="async.concurrency"): + create_config(collected) + + +def test_collect_config_unknown_empty_mapping_warns( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config_file = tmp_path / "zarr.yaml" + config_file.write_text("future: {}\n") + monkeypatch.setenv("ZARR_CONFIG", str(config_file)) + with pytest.warns(ZarrUserWarning, match="future"): + collected = collect_config() + assert "future" not in collected + + +@pytest.mark.parametrize("key", ["config", "root_config", "benchmark_clear_cache", "future"]) +def test_collect_config_warns_for_control_names_in_yaml( + key: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config_file = tmp_path / "zarr.yaml" + config_file.write_text(f"{key}: 1\narray:\n order: F\n") + monkeypatch.setenv("ZARR_CONFIG", str(config_file)) + with pytest.warns(ZarrUserWarning, match=key): + collected = collect_config() + assert key not in collected + assert create_config(collected).array.order == "F" + + +def test_create_config_is_independent_of_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ZARR_ARRAY__ORDER", "F") + assert create_config({}).array.order == "C" + cfg = create_config({"array.order": "F", "codecs.custom": "pkg.Custom"}) + assert cfg.array.order == "F" + assert cfg.codecs["custom"] == "pkg.Custom" + + +@pytest.mark.parametrize("key", ["benchmark_clear_cache", "array.ordre", "future.key"]) +def test_create_config_unknown_key_raises(key: str) -> None: + with pytest.raises(KeyError): + create_config({key: 1}) + + +@pytest.mark.parametrize( + ("key", "value"), + [ + ("array.order", "Q"), + ("async.concurrency", "many"), + ("array.write_empty_chunks", "false"), + ("codecs.custom", 1), + ], +) +def test_create_config_invalid_value_raises(key: str, value: object) -> None: + with pytest.raises(ValueError, match=key): + create_config({key: value}) diff --git a/tests/test_config_compatibility.py b/tests/test_config_compatibility.py new file mode 100644 index 0000000000..67cc6bf24a --- /dev/null +++ b/tests/test_config_compatibility.py @@ -0,0 +1,179 @@ +"""Compatibility at the public config API and donfig reader boundaries.""" + +from __future__ import annotations + +import copy +import dataclasses +import os +from collections.abc import Iterator, Mapping +from pathlib import Path +from tempfile import TemporaryDirectory +from types import UnionType +from typing import Any, get_args, get_origin, get_type_hints + +import donfig +import pytest +import yaml +from hypothesis import given +from hypothesis import strategies as st + +from zarr.core.config import ( + _SERIALIZED_NAMES, + ZarrConfigManager, + collect_environment, + make_default_config, +) + + +def _config_values(annotation: Any) -> st.SearchStrategy[object]: + """Use field types, with bounded numbers and strings safe for env/YAML transport.""" + if annotation is int: + return st.integers(min_value=1, max_value=256) + if annotation is float: + return st.floats(min_value=0, max_value=256, allow_nan=False, allow_infinity=False) + if annotation is str: + return st.from_regex(r"custom\.[A-Za-z][A-Za-z0-9]{0,10}", fullmatch=True) + if get_origin(annotation) is UnionType: + return st.one_of(*(_config_values(member) for member in get_args(annotation))) + # Includes bool, NoneType, and Literal fields such as order and Zarr format. + return st.from_type(annotation) + + +def _config_leaves( + node: object, prefix: tuple[str, ...] = () +) -> Iterator[tuple[tuple[str, ...], st.SearchStrategy[object]]]: + """Discover schema leaves and registered default codec names, retaining literal dots.""" + if isinstance(node, Mapping): + for key, value in node.items(): + yield (*prefix, key), _config_values(type(value)) + else: + assert dataclasses.is_dataclass(node) + assert not isinstance(node, type) + hints = get_type_hints(type(node)) + for field in dataclasses.fields(node): + path = (*prefix, _SERIALIZED_NAMES.get(field.name, field.name)) + value = getattr(node, field.name) + if dataclasses.is_dataclass(value) or isinstance(value, Mapping): + yield from _config_leaves(value, path) + else: + yield path, _config_values(hints[field.name]) + + +def _config_cases(*, dotted_api: bool) -> st.SearchStrategy[tuple[tuple[str, ...], object, object]]: + # Donfig's dotted API cannot address a literal dot inside a codec name. + # YAML tests retain these leaves because YAML preserves mapping boundaries. + return st.one_of( + *( + st.tuples(st.just(path), values, values) + for path, values in _config_leaves(make_default_config()) + if not dotted_api or all("." not in part for part in path) + ) + ) + + +def _alternate_key(key: str) -> str: + return ".".join( + part.replace("_", "-") if "_" in part else part.replace("-", "_") for part in key.split(".") + ) + + +@given(case=_config_cases(dotted_api=True)) +def test_collect_environment_recognizes_schema_keys( + case: tuple[tuple[str, ...], object, object], +) -> None: + path, value, _ = case + name = "ZARR_" + "__".join(path).replace("-", "_").upper() + config_environment = {name: repr(value)} + controls = {"ZARR_BENCHMARK_CLEAR_CACHE": "1"} + assert collect_environment(config_environment | controls) == { + "config": config_environment, + "controls": controls, + } + + +@given(case=_config_cases(dotted_api=True), alternate=st.booleans()) +def test_config_key_spellings_match_donfig( + case: tuple[tuple[str, ...], object, object], alternate: bool +) -> None: + """Aliases select the same leaf and scoped updates restore the entire config.""" + path, selected, _ = case + key = ".".join(path) + cfg = ZarrConfigManager() + reference = donfig.Config("zarr", defaults=[cfg.to_dict()], paths=[], env={}) + alias = _alternate_key(key) + written_key = alias if alternate else key + before = cfg.to_dict() + with reference.set({written_key: selected}), cfg.set({written_key: selected}): + assert cfg.get(key) == reference.get(key) + assert cfg.get(alias) == reference.get(alias) + assert alias in cfg + assert cfg.to_dict() == reference.config + assert cfg.to_dict() == reference.config == before + + +@given(case=_config_cases(dotted_api=True)) +def test_config_nested_kwargs_match_donfig(case: tuple[tuple[str, ...], object, object]) -> None: + """Nested kwargs override mapping entries, including when their spellings differ.""" + path, first, second = case + key = ".".join(path) + cfg = ZarrConfigManager() + reference = donfig.Config("zarr", defaults=[cfg.to_dict()], paths=[], env={}) + keyword = key.replace("-", "_").replace(".", "__") + before = cfg.to_dict() + with ( + reference.set({key: first}, **{keyword: second}), + cfg.set({key: first}, **{keyword: second}), + ): + assert cfg.get(key) == second + assert cfg.to_dict() == reference.config + assert cfg.to_dict() == reference.config == before + + +@given(case=_config_cases(dotted_api=False), env_override=st.booleans()) +def test_config_yaml_aliases_match_donfig( + case: tuple[tuple[str, ...], object, object], env_override: bool +) -> None: + """Hyphen/underscore YAML spellings retain defaults and environment precedence.""" + path, selected, override = case + key = ".".join(path) + with TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as patch: + for name in list(os.environ): + if name.startswith("ZARR_"): + patch.delenv(name) + defaults = ZarrConfigManager().defaults + data: object = selected + for part in reversed(path): + data = {_alternate_key(part): data} + config_file = Path(directory) / "zarr.yaml" + config_file.write_text(yaml.safe_dump(data)) + patch.setenv("ZARR_CONFIG", str(config_file)) + # Donfig's environment reader also interprets literal dots as nesting. + if env_override and all("." not in part for part in path): + env_key = "ZARR_" + "__".join(path).replace("-", "_").upper() + selected = override + patch.setenv(env_key, repr(selected)) + reference = donfig.Config("zarr", defaults=[defaults]) + reference.config.pop("config", None) + cfg = ZarrConfigManager() + assert cfg.get(key) == selected + assert cfg.to_dict() == reference.config + + +@given(case=_config_cases(dotted_api=True)) +def test_config_manager_deepcopy_is_independent( + case: tuple[tuple[str, ...], object, object], +) -> None: + """Copy the public manager, including its lock, rather than just its snapshot.""" + path, value, replacement = case + key = ".".join(path) + cfg = ZarrConfigManager() + cfg.set({key: value, "codecs.custom": "custom.Original"}) + restored = copy.deepcopy(cfg) + assert restored.to_dict() == cfg.to_dict() + with restored.set({key: replacement}): + assert restored.get(key) == replacement + assert cfg.get(key) == value + assert restored.get(key) == value + # The currently mutable subtree must also be deeply copied. + restored.get("codecs")["custom"] = "custom.Copy" + assert cfg.get("codecs.custom") == "custom.Original" diff --git a/tests/test_config_typed.py b/tests/test_config_typed.py new file mode 100644 index 0000000000..3a1a674695 --- /dev/null +++ b/tests/test_config_typed.py @@ -0,0 +1,919 @@ +from __future__ import annotations + +import copy +import dataclasses +import os +import pickle +import threading +import typing +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from tests.conftest import Expect, ExpectFail +from zarr.core.config import ( + _SERIALIZED_NAMES, + DEFAULT_CODECS, + BadConfigError, + ZarrConfig, + ZarrConfigManager, + apply_overrides, + build_config, + get_path, + make_default_config, + replace_path, + to_nested_dict, +) + +if typing.TYPE_CHECKING: + import pathlib + +# --------------------------------------------------------------------------- +# Module-level constants used in parametrize lists (evaluated at collection time) +# --------------------------------------------------------------------------- + +_REMOVED_KEY = "array.v2_default_compressor.numeric" +_DEFAULT = make_default_config() + + +def _build_config_with_env(monkeypatch: pytest.MonkeyPatch, env: dict[str, str]) -> ZarrConfig: + """Run `build_config` under a controlled ``ZARR_*`` environment. + + `build_config` delegates env/YAML reading to donfig, which reads + ``os.environ``. This clears any ambient ``ZARR_*`` variables for determinism, + sets the requested ones, then builds. + """ + for name in list(os.environ): + if name.startswith("ZARR_"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + return build_config() + + +# --------------------------------------------------------------------------- +# 1. get_path — success cases +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input="array.order", output="C", id="array-order"), + Expect(input="async.concurrency", output=10, id="async-concurrency-alias"), + Expect(input="json_indent", output=2, id="json-indent"), + Expect(input="codecs", output=DEFAULT_CODECS, id="codecs-dict"), + Expect(input="codecs.blosc", output="zarr.codecs.blosc.BloscCodec", id="codecs-blosc"), + ], + ids=lambda c: c.id, +) +def test_get_path(case: Expect[str, object]) -> None: + assert get_path(make_default_config(), case.input) == case.output + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail(input="array.nonexistent", exception=KeyError, id="nonexistent-key"), + ], + ids=lambda c: c.id, +) +def test_get_path_raises(case: ExpectFail[str]) -> None: + with case.raises(): + get_path(make_default_config(), case.input) + + +# --------------------------------------------------------------------------- +# 2. replace_path +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input=("array.order", "F"), output="F", id="array-order"), + Expect(input=("async.concurrency", 99), output=99, id="async-concurrency-alias"), + Expect( + input=("codecs.my_codec", "my.module.MyCodec"), + output="my.module.MyCodec", + id="codec-new-key", + ), + ], + ids=lambda c: c.id, +) +def test_replace_path(case: Expect[tuple[str, object], object]) -> None: + key, value = case.input + result = replace_path(make_default_config(), key, value) + assert get_path(result, key) == case.output + + +def test_replace_path_is_immutable() -> None: + """Original config is unchanged after replace_path (frozen dataclass).""" + cfg = make_default_config() + _ = replace_path(cfg, "array.order", "F") + assert cfg.array.order == "C" + # the open `codecs` dict must not be mutated in place either: a frozen + # dataclass forbids attribute re-assignment but not `dict.__setitem__`. + _ = replace_path(cfg, "codecs.my_codec", "my.module.MyCodec") + assert "my_codec" not in cfg.codecs + + +# --------------------------------------------------------------------------- +# 3. build_config — env ingest (donfig reads ZARR_* from the environment) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input={}, output=_DEFAULT, id="empty-environ"), + Expect( + input={"ZARR_CONFIG": "/nonexistent/path.yaml"}, + output=_DEFAULT, + id="zarr-config-nonexistent", + ), + Expect( + input={"ZARR_JSON_INDENT": "4"}, + output=replace_path(_DEFAULT, "json_indent", 4), + id="json-indent-env", + ), + Expect( + input={ + "ZARR_ARRAY__ORDER": "F", + "ZARR_ASYNC__CONCURRENCY": "32", + "ZARR_CODECS__MY_CODEC": "my.module.MyCodec", + }, + output=replace_path( + replace_path( + replace_path(_DEFAULT, "array.order", "F"), + "async.concurrency", + 32, + ), + "codecs.my_codec", + "my.module.MyCodec", + ), + id="nested-literal-and-codecs", + ), + ], + ids=lambda c: c.id, +) +def test_build_config_env( + case: Expect[dict[str, str], ZarrConfig], monkeypatch: pytest.MonkeyPatch +) -> None: + """`ZARR_*` environment variables are read (via donfig) and applied onto the + typed defaults: dotted nesting (`ZARR_ARRAY__ORDER`), literal parsing + (`ZARR_ASYNC__CONCURRENCY=32` -> int), and the open `codecs` namespace all + work; a `ZARR_CONFIG` pointing nowhere is a no-op.""" + assert _build_config_with_env(monkeypatch, case.input) == case.output + + +# --------------------------------------------------------------------------- +# 5. apply_overrides +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect( + input={"array.order": "F", "codecs.x": "pkg.X"}, + output=replace_path(replace_path(_DEFAULT, "array.order", "F"), "codecs.x", "pkg.X"), + id="array-order-and-codec", + ), + ], + ids=lambda c: c.id, +) +def test_apply_overrides(case: Expect[dict[str, object], ZarrConfig]) -> None: + assert apply_overrides(make_default_config(), case.input) == case.output + + +# --------------------------------------------------------------------------- +# 6. to_nested_dict +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect( + input=make_default_config(), + output=("C", 10, "zarr.codecs.blosc.BloscCodec"), + id="default-serialized-keys", + ), + ], + ids=lambda c: c.id, +) +def test_to_nested_dict(case: Expect[ZarrConfig, tuple[str, int, str]]) -> None: + nested = to_nested_dict(case.input) + order, concurrency, blosc = case.output + assert nested["array"]["order"] == order + assert nested["async"]["concurrency"] == concurrency + assert "async_" not in nested # serialized key, not the Python attribute name + assert nested["codecs"]["blosc"] == blosc + + +# --------------------------------------------------------------------------- +# 7. ZarrConfigManager.get — proxy string access +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input="array.order", output="C", id="array-order"), + Expect(input="async.concurrency", output=10, id="async-concurrency-alias"), + Expect(input="codecs", output=DEFAULT_CODECS, id="codecs-dict"), + ], + ids=lambda c: c.id, +) +def test_proxy_get(case: Expect[str, object]) -> None: + assert ZarrConfigManager().get(case.input) == case.output + + +@pytest.mark.parametrize( + "case", + [ + Expect(input=("does.not.exist", "fallback"), output="fallback", id="default-fallback"), + ], + ids=lambda c: c.id, +) +def test_proxy_get_with_default(case: Expect[tuple[str, object], object]) -> None: + key, default = case.input + assert ZarrConfigManager().get(key, default) == case.output + + +# --------------------------------------------------------------------------- +# 8. Removed-deprecated-key behavior +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + Expect(input="fallback", output="fallback", id="get-with-default"), + ], + ids=lambda c: c.id, +) +def test_removed_deprecated_key_get_default(case: Expect[str, str]) -> None: + """get() with a removed deprecated key and a default returns the default silently.""" + assert ZarrConfigManager().get(_REMOVED_KEY, case.input) == case.output + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail(input=_REMOVED_KEY, exception=KeyError, id="get-no-default"), + ], + ids=lambda c: c.id, +) +def test_removed_deprecated_key_get_raises(case: ExpectFail[str]) -> None: + """get() with a removed deprecated key and no default raises KeyError.""" + with case.raises(): + ZarrConfigManager().get(case.input) + + +# --------------------------------------------------------------------------- +# 9. set() must raise for both removed-deprecated keys and totally unknown keys +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail( + input={_REMOVED_KEY: "some_value"}, + exception=BadConfigError, + id="set-removed-deprecated", + ), + ExpectFail( + input={"totally.bogus.key": 1}, + exception=KeyError, + id="set-unknown-key", + ), + ], + ids=lambda c: c.id, +) +def test_set_invalid_key_raises(case: ExpectFail[dict[str, object]]) -> None: + """set() raises for both removed deprecated keys and totally unknown structured keys.""" + with case.raises(): + ZarrConfigManager().set(case.input) + + +# --------------------------------------------------------------------------- +# 10. Unknown keys produce a helpful "did you mean" message (get and set) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "case", + [ + # close match at the deepest resolvable level -> "Did you mean ...?" + ExpectFail( + input="arr4y", exception=KeyError, msg=r"Did you mean .array.", id="suggest-top" + ), + ExpectFail( + input="array.0rder", + exception=KeyError, + msg=r"Did you mean .array\.order.", + id="suggest-nested", + ), + ExpectFail( + input="codecs.bl0sc", + exception=KeyError, + msg=r"Did you mean .codecs\.blosc.", + id="suggest-codec", + ), + # no close match -> roster of available keys at the last resolvable level + ExpectFail(input="foo", exception=KeyError, msg=r"Valid keys: .*array", id="roster-top"), + ExpectFail( + input="array.foo", + exception=KeyError, + msg=r"Valid keys under .array.: .*order", + id="roster-nested", + ), + ExpectFail( + input="codecs.zzzzzzzz", + exception=KeyError, + msg=r"under .codecs.: .*more\)", + id="roster-truncated", + ), + ], + ids=lambda c: c.id, +) +def test_get_unknown_key_message(case: ExpectFail[str]) -> None: + """get() on an unknown key suggests the closest key or lists what's available.""" + with case.raises(): + ZarrConfigManager().get(case.input) + + +@pytest.mark.parametrize( + "case", + [ + ExpectFail( + input={"array.0rder": "F"}, + exception=KeyError, + msg=r"Did you mean .array\.order.", + id="set-suggest", + ), + ExpectFail( + input={"array.foo": "F"}, + exception=KeyError, + msg=r"Valid keys under .array.: .*order", + id="set-roster", + ), + ], + ids=lambda c: c.id, +) +def test_set_unknown_key_message(case: ExpectFail[dict[str, object]]) -> None: + """set() shares the same helpful unknown-key error as get().""" + with case.raises(): + ZarrConfigManager().set(case.input) + + +# --------------------------------------------------------------------------- +# Default config values (dedicated — direct attribute assertions are clearest here) +# --------------------------------------------------------------------------- + + +def test_default_config_values() -> None: + cfg = make_default_config() + assert cfg.default_zarr_format == 3 + assert cfg.array.order == "C" + assert cfg.array.sharding_coalesce_max_bytes == 16 << 20 + assert cfg.async_.concurrency == 10 + assert cfg.async_.timeout is None + assert cfg.threading.max_workers is None + assert cfg.json_indent == 2 + assert cfg.codec_pipeline.path == "zarr.core.codec_pipeline.BatchedCodecPipeline" + assert cfg.codecs["blosc"] == "zarr.codecs.blosc.BloscCodec" + assert cfg.codecs == DEFAULT_CODECS + # proxy attribute access via ZarrConfigManager + mgr = ZarrConfigManager() + assert mgr.array.order == "C" + + +# --------------------------------------------------------------------------- +# Stateful / behavioral tests (kept as dedicated functions) +# --------------------------------------------------------------------------- + + +def test_set_permanent_and_context() -> None: + cfg = ZarrConfigManager() + cfg.set({"array.order": "F"}) + assert cfg.get("array.order") == "F" # permanent + with cfg.set({"array.order": "C"}): + assert cfg.get("array.order") == "C" + assert cfg.get("array.order") == "F" # restored to permanent value + cfg.reset() + assert cfg.get("array.order") == "C" + + +def test_permanent_set_visible_in_worker_thread() -> None: + cfg = ZarrConfigManager() + cfg.set({"async.concurrency": 77}) + try: + with ThreadPoolExecutor(max_workers=1) as ex: + seen = ex.submit(lambda: cfg.get("async.concurrency")).result() + assert seen == 77 # a permanent set is on the shared global base + finally: + cfg.reset() + + +def test_permanent_set_cross_thread_last_writer_wins() -> None: + """A permanent `set` from any thread updates the shared global base, so a later + permanent `set` in another thread is visible everywhere — even after the first + thread already did its own permanent `set`. (Regression: the first set used to + pin the setting thread's view, so it kept seeing its own stale value.)""" + cfg = ZarrConfigManager() + cfg.set({"async.concurrency": 1}) + worker = threading.Thread(target=lambda: cfg.set({"async.concurrency": 999})) + worker.start() + worker.join() + assert cfg.get("async.concurrency") == 999 + + +def test_with_block_is_global_and_reverts_only_its_keys() -> None: + """A `with config.set(...)` applies globally (donfig semantics: it is NOT + isolated to the calling thread) and, on exit, reverts only the keys it set — + leaving a concurrent permanent `set` to a *different* key intact.""" + cfg = ZarrConfigManager() + cfg.set({"async.concurrency": 5}) + with cfg.set({"async.concurrency": 999}): + assert cfg.get("async.concurrency") == 999 # visible in this context + with ThreadPoolExecutor(max_workers=1) as ex: + seen = ex.submit(lambda: cfg.get("async.concurrency")).result() + assert seen == 999 # globally visible: the worker sees the override too + # a concurrent permanent set to a DIFFERENT key must survive block exit + cfg.set({"array.order": "F"}) + assert cfg.get("async.concurrency") == 5 # the block's key reverted + assert cfg.get("array.order") == "F" # the other key was not clobbered + + +def test_permanent_set_inside_with_block_persists() -> None: + """On `with config.set(...)` exit, only the block's own keys are reverted, so a + permanent `set` to a different key made inside the block persists.""" + cfg = ZarrConfigManager() + with cfg.set({"array.order": "F"}): + cfg.set({"async.concurrency": 5}) # permanent set to a different key + assert cfg.get("array.order") == "C" # the block's key reverted + assert cfg.get("async.concurrency") == 5 # the other key persisted + + +def test_with_block_removes_newly_added_codec_key_on_exit() -> None: + """A scoped `set` that *adds* a new codec key removes it again on block exit, + while a pre-existing key it overrode is restored to its prior value.""" + cfg = ZarrConfigManager() + assert "brand_new_codec" not in cfg.codecs + with cfg.set({"codecs.brand_new_codec": "pkg.New", "codecs.blosc": "pkg.OverrideBlosc"}): + assert cfg.codecs["brand_new_codec"] == "pkg.New" + assert cfg.codecs["blosc"] == "pkg.OverrideBlosc" + assert "brand_new_codec" not in cfg.codecs # newly added key removed + assert cfg.codecs["blosc"] == DEFAULT_CODECS["blosc"] # existing key restored + + +# --------------------------------------------------------------------------- +# Subtree item access (donfig back-compat for `config.get("array")["order"]`) +# --------------------------------------------------------------------------- + + +def test_subtree_get_item_access_matches_attribute() -> None: + """A subtree `get` returns a typed dataclass that also supports donfig-style + item access: `["order"]`, `.order`, and `get("array.order")` all agree, and + an unknown key raises `KeyError` like the old dicts did.""" + cfg = ZarrConfigManager() + array = cfg.get("array") + assert array["order"] == array.order == cfg.get("array.order") + assert array["sharding_coalesce_max_bytes"] == array.sharding_coalesce_max_bytes + with pytest.raises(KeyError): + array["does_not_exist"] + + +def test_config_node_dotted_and_alias_item_access() -> None: + """Item access on a config node resolves dotted keys, the `async` alias, and + the open `codecs` mapping, mirroring `get_path`.""" + cfg = make_default_config() + assert cfg["array.order"] == "C" + assert cfg["async.concurrency"] == 10 # the `async` alias resolves + assert cfg.async_["concurrency"] == cfg.async_.concurrency # node item access + assert cfg["codecs.bytes"] == DEFAULT_CODECS["bytes"] + + +def test_defaults_and_enable_gpu() -> None: + cfg = ZarrConfigManager() + assert cfg.defaults["array"]["order"] == "C" + with cfg.set({"buffer": "x"}): + pass + cfg.enable_gpu() + try: + assert cfg.get("buffer") == "zarr.buffer.gpu.Buffer" + assert cfg.get("ndbuffer") == "zarr.buffer.gpu.NDBuffer" + finally: + cfg.reset() + + +def test_refresh_not_shadowed_by_prior_scope(monkeypatch: pytest.MonkeyPatch) -> None: + """refresh() must be visible in the calling context even after a prior set()/reset().""" + mgr = ZarrConfigManager() + # apply a permanent override in this context, then rebuild over a changed env + mgr.set({"array.order": "F"}) + assert mgr.get("array.order") == "F" + # change the environment so a rebuild differs, then refresh + monkeypatch.setenv("ZARR_JSON_INDENT", "7") + mgr.refresh() + # refresh rebuilds the global base and must be visible in THIS context + assert mgr.get("json_indent") == 7 + assert mgr.get("array.order") == "C" # the prior permanent set is gone after rebuild + + +# --------------------------------------------------------------------------- +# Tolerant ingest: unknown env/YAML keys must warn and be skipped, not crash +# --------------------------------------------------------------------------- + + +def test_build_config_unknown_env_key_warns_and_skips(monkeypatch: pytest.MonkeyPatch) -> None: + """build_config with an unrecognized env var warns and skips it; known keys still apply.""" + with pytest.warns(UserWarning, match="future.key"): + cfg = _build_config_with_env( + monkeypatch, {"ZARR_FUTURE__KEY": "1", "ZARR_ARRAY__ORDER": "F"} + ) + # Known key was applied + assert cfg.array.order == "F" + # All other fields are still at default + default = make_default_config() + from dataclasses import fields as dc_fields + + for f in dc_fields(default): + if f.name != "array": + assert getattr(cfg, f.name) == getattr(default, f.name) + + +def test_apply_overrides_unknown_key_raises() -> None: + """Unknown keys must be filtered at collection, not silently dropped at construction.""" + default = make_default_config() + with pytest.raises(KeyError, match="totally.bogus.key"): + apply_overrides(default, {"totally.bogus.key": 123}) + + +# --------------------------------------------------------------------------- +# donfig is used as the env/YAML reader +# --------------------------------------------------------------------------- + + +def test_donfig_is_used_for_ingest() -> None: + """donfig backs env/YAML ingest, so building a config imports it and its + reader produces a mapping we can consume.""" + import donfig + + assert isinstance(donfig.Config("zarr").config, dict) + # build_config runs donfig's reader and returns the typed representation + assert isinstance(build_config(), ZarrConfig) + + +# --------------------------------------------------------------------------- +# YAML config files (read by donfig, applied onto the typed defaults) +# --------------------------------------------------------------------------- + + +def test_yaml_codecs_block_merges_not_replaces( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A YAML file with a codecs: block must MERGE into the defaults, not replace them.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("codecs:\n bytes: my.custom.BytesCodec\n mycodec: my.Mod.MyCodec\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + # overrides applied + assert cfg.codecs["bytes"] == "my.custom.BytesCodec" + assert cfg.codecs["mycodec"] == "my.Mod.MyCodec" + # defaults PRESERVED + assert cfg.codecs["blosc"] == "zarr.codecs.blosc.BloscCodec" + assert cfg.codecs["zstd"] == "zarr.codecs.zstd.ZstdCodec" + # exactly one net-new key added ("bytes" overwrites existing; "mycodec" is new) + assert len(cfg.codecs) == len(DEFAULT_CODECS) + 1 + + +def test_yaml_dotted_codec_name_merges( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Dotted codec keys like numcodecs.bz2 in YAML must merge, not replace the whole dict.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("codecs:\n numcodecs.bz2: my.Override\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + # dotted key correctly round-tripped + assert cfg.codecs["numcodecs.bz2"] == "my.Override" + # all other defaults preserved + assert cfg.codecs["blosc"] == "zarr.codecs.blosc.BloscCodec" + assert len(cfg.codecs) == len(DEFAULT_CODECS) # bz2 was already there; just overwritten + + +def test_yaml_file_via_zarr_config_is_read( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A YAML file pointed to by ZARR_CONFIG is read; a non-existent path is a no-op.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("json_indent: 9\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + assert cfg.json_indent == 9 + # Non-existent path must still not raise, and leaves defaults intact + cfg2 = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": "/nonexistent/path.yaml"}) + assert cfg2.json_indent == make_default_config().json_indent + + +def test_yaml_config_directory_is_scanned( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ZARR_CONFIG may point at a directory; a zarr.yaml inside it is loaded.""" + (tmp_path / "zarr.yaml").write_text("array:\n order: F\njson_indent: 8\n") + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(tmp_path)}) + assert cfg.array.order == "F" + assert cfg.json_indent == 8 + + +def test_env_var_overrides_yaml(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Precedence is defaults < YAML < environment: an env var wins over a YAML file.""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("json_indent: 3\n") + cfg = _build_config_with_env( + monkeypatch, {"ZARR_CONFIG": str(yaml_file), "ZARR_JSON_INDENT": "9"} + ) + assert cfg.json_indent == 9 + + +def test_yaml_unknown_key_warns_and_skips( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An unrecognized key in a YAML file warns and is skipped; known keys still apply + (so a version-skewed config file can't prevent `import zarr`).""" + yaml_file = tmp_path / "zarr.yaml" + yaml_file.write_text("future_key: 1\narray:\n order: F\n") + with pytest.warns(UserWarning, match="future_key"): + cfg = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(yaml_file)}) + assert cfg.array.order == "F" + + +def test_discovers_home_config_dir(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A `zarr.yaml` in `~/.config/zarr` is discovered (donfig's primary location), + and `ZARR_CONFIG` takes precedence over it while home-only keys still apply. + + Regression guard against silently dropping donfig's config-file locations. + """ + home = tmp_path / "home" + config_dir = home / ".config" / "zarr" + config_dir.mkdir(parents=True) + (config_dir / "zarr.yaml").write_text("json_indent: 1\narray:\n order: F\n") + # Redirect `~` to the temporary home so the real user config is not consulted; + # donfig computes its `~/.config/zarr` path via os.path.expanduser. + monkeypatch.setattr(os.path, "expanduser", lambda p: str(home) if p == "~" else p) + + # Without ZARR_CONFIG, the home config directory is picked up. + cfg = _build_config_with_env(monkeypatch, {}) + assert cfg.json_indent == 1 + assert cfg.array.order == "F" + + # ZARR_CONFIG (highest precedence) overrides the home file's overlapping keys. + override = tmp_path / "override.yaml" + override.write_text("json_indent: 2\n") + cfg2 = _build_config_with_env(monkeypatch, {"ZARR_CONFIG": str(override)}) + assert cfg2.json_indent == 2 # ZARR_CONFIG wins + assert cfg2.array.order == "F" # home-only key still applies + + +# --------------------------------------------------------------------------- +# Drift-protection: every structured leaf key must have a get() overload +# --------------------------------------------------------------------------- + + +def _structured_leaf_specs(cfg_cls: type, prefix: str = "") -> dict[str, object]: + """Walk a settings dataclass recursively and return ``{dotted_key: resolved_type}``. + + Uses ``typing.get_type_hints`` instead of ``f.type`` so that the + ``from __future__ import annotations`` string-annotation form is resolved + to real types before ``dataclasses.is_dataclass`` is called. The open + ``codecs`` mapping is intentionally excluded. + """ + specs: dict[str, object] = {} + resolved_hints = typing.get_type_hints(cfg_cls) + for f in dataclasses.fields(cfg_cls): + serialized = _SERIALIZED_NAMES.get(f.name, f.name) + key = f"{prefix}.{serialized}" if prefix else serialized + resolved_type = resolved_hints[f.name] + if dataclasses.is_dataclass(resolved_type): + specs.update(_structured_leaf_specs(typing.cast(type, resolved_type), key)) + elif f.name == "codecs": + # open mapping — intentionally not enumerated + continue + else: + specs[key] = resolved_type + return specs + + +def _structured_leaf_keys(cfg_cls: type, prefix: str = "") -> list[str]: + """Return every dotted leaf key for a settings dataclass (derived from specs).""" + return list(_structured_leaf_specs(cfg_cls, prefix)) + + +def test_every_structured_key_has_a_get_overload() -> None: + """Enumerate every typed leaf key in ZarrConfig and assert a matching get() overload exists.""" + overloads = typing.get_overloads(ZarrConfigManager.get) + literal_keys: set[str] = set() + for ov in overloads: + hints = typing.get_type_hints(ov) + key_hint = hints.get("key") + if typing.get_origin(key_hint) is typing.Literal: + literal_keys.update(typing.get_args(key_hint)) + leaf_keys = _structured_leaf_keys(ZarrConfig) + missing = set(leaf_keys) - literal_keys + assert not missing, f"get() overloads missing for: {sorted(missing)}" + + +def test_get_overload_return_types_match_fields() -> None: + """Assert that each get() overload's return type matches the dataclass field type. + + Builds two maps using ``typing.get_type_hints`` — one from the dataclass + field annotations, one from the overload return hints — then compares them + key by key. A mismatch (e.g. ``-> str`` instead of ``-> Literal["C","F"]``) + is reported as a clear failure rather than a missing-overload failure. + """ + # Build map: key -> return type from overloads + overloads = typing.get_overloads(ZarrConfigManager.get) + overload_return: dict[str, object] = {} + for ov in overloads: + hints = typing.get_type_hints(ov) + key_hint = hints.get("key") + if typing.get_origin(key_hint) is typing.Literal: + (literal_val,) = typing.get_args(key_hint) + overload_return[literal_val] = hints["return"] + + # Build map: key -> field type from the dataclass schema + field_specs = _structured_leaf_specs(ZarrConfig) + + missing: list[str] = [] + mismatched: list[str] = [] + for key, expected_type in field_specs.items(): + if key not in overload_return: + missing.append(f" {key!r}: missing overload") + elif overload_return[key] != expected_type: + mismatched.append( + f" {key!r}: overload returns {overload_return[key]!r}," + f" field type is {expected_type!r}" + ) + + errors: list[str] = [] + if missing: + errors.append("get() overloads missing for keys:\n" + "\n".join(missing)) + if mismatched: + errors.append( + "get() overload return types do not match field types:\n" + "\n".join(mismatched) + ) + assert not errors, "\n\n".join(errors) + + +# --------------------------------------------------------------------------- +# Regression tests for the #4101 review +# --------------------------------------------------------------------------- + + +def test_env_codec_override_canonicalizes_hyphenated_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`ZARR_CODECS__VLEN_UTF8` must override the hyphenated default `vlen-utf8`. + + Environment variables can't contain hyphens, so the flattened key is + `codecs.vlen_utf8`; without canonicalization it lands under a dead key while + the registry keeps reading the untouched `vlen-utf8` default. + """ + cfg = _build_config_with_env(monkeypatch, {"ZARR_CODECS__VLEN_UTF8": "my.Override"}) + assert cfg.codecs["vlen-utf8"] == "my.Override" + assert "vlen_utf8" not in cfg.codecs # no dead underscore key + # underscore-named defaults (e.g. sharding_indexed) are untouched + cfg2 = _build_config_with_env(monkeypatch, {"ZARR_CODECS__SHARDING_INDEXED": "my.Shard"}) + assert cfg2.codecs["sharding_indexed"] == "my.Shard" + # a brand-new codec name with underscores stays as written + cfg3 = _build_config_with_env(monkeypatch, {"ZARR_CODECS__MY_NEW": "my.New"}) + assert cfg3.codecs["my_new"] == "my.New" + + +@pytest.mark.parametrize("key", ["array.order.upper", "default_zarr_format.numerator"]) +def test_get_does_not_descend_into_scalar_attributes(key: str) -> None: + """A dotted key that walks past a scalar leaf must raise, not resolve a stray + Python attribute (e.g. `str.upper` / `int.numerator`).""" + with pytest.raises(KeyError): + ZarrConfigManager().get(key) + + +def test_set_rejects_descending_into_scalar() -> None: + with pytest.raises(KeyError): + ZarrConfigManager().set({"array.order.upper": "X"}) + + +def test_codecs_public_mapping_is_read_only() -> None: + """`config.codecs` exposes a read-only view, so a live snapshot can't be + mutated in place; a mutable copy is still available via `dict(...)`.""" + cfg = ZarrConfigManager() + with pytest.raises(TypeError): + cfg.codecs["blosc"] = "x" # type: ignore[index] + assert dict(cfg.codecs)["blosc"] == cfg.codecs["blosc"] + + +def test_config_snapshot_is_picklable_and_deepcopyable() -> None: + """A `ZarrConfig` snapshot must stay picklable / deep-copyable: the `codecs` + field is a plain dict, and only the public view is a read-only proxy.""" + cfg = replace_path(make_default_config(), "codecs.x", "pkg.X") + assert pickle.loads(pickle.dumps(cfg)) == cfg + assert copy.deepcopy(cfg) == cfg + + +def test_set_structured_subtree_dict_is_rejected() -> None: + """Assigning a dict to a whole structured subtree is rejected (it would drop + sibling fields and break attribute access); leaf keys must be used instead.""" + cfg = ZarrConfigManager() + with pytest.raises(TypeError): + cfg.set({"array": {"order": "F"}}) + # the leaf-key form works and preserves siblings + cfg.set({"array.order": "F"}) + assert cfg.get("array.order") == "F" + assert cfg.get("array.write_empty_chunks") is False + + +def test_manager_is_not_iterable() -> None: + """Iterating the manager raises a clear TypeError rather than falling into the + legacy integer-index protocol (which gave a confusing error).""" + with pytest.raises(TypeError): + list(ZarrConfigManager()) + + +def test_subtree_node_supports_mapping_style_reads() -> None: + """A subtree returned by `get` supports donfig-style `in`, iteration, `keys`, + and `dict()` alongside attribute and item access.""" + array = ZarrConfigManager().get("array") + assert "order" in array + assert "bogus" not in array + assert "order" in list(array) + assert "order" in set(array.keys()) + assert dict(array)["order"] == array.order == "C" + assert len(array) == len(dataclasses.fields(array)) + + +def test_manager_item_and_membership_access() -> None: + """donfig-style `config["k"]` and `"k" in config` mirror `get`.""" + cfg = ZarrConfigManager() + assert cfg["array.order"] == cfg.get("array.order") + assert "array.order" in cfg + assert "bogus.key" not in cfg + assert 123 not in cfg # non-string key is absent, not an error + + +def test_config_alias_preserved() -> None: + """`Config` remains importable as an alias of `ZarrConfigManager` (pre-typed + imports and `isinstance` checks keep working).""" + from zarr.core.config import Config + + assert Config is ZarrConfigManager + assert isinstance(ZarrConfigManager(), Config) + + +def test_concurrent_permanent_sets_to_distinct_keys_all_survive() -> None: + """A permanent `set` rebuilds the whole snapshot from `_base`; the manager + locks that read-modify-write so concurrent sets to distinct keys don't lose + updates.""" + cfg = ZarrConfigManager() + n = 64 + barrier = threading.Barrier(n) + + def worker(i: int) -> None: + barrier.wait() + cfg.set({f"codecs.k{i}": f"v{i}"}) + + with ThreadPoolExecutor(max_workers=n) as ex: + list(ex.map(worker, range(n))) + + codecs = cfg.get("codecs") + assert all(codecs.get(f"k{i}") == f"v{i}" for i in range(n)) + + +# --------------------------------------------------------------------------- +# Static-typing smoke test (only checked by mypy, not executed at runtime) +# --------------------------------------------------------------------------- + +if typing.TYPE_CHECKING: + + def _typing_smoke(cfg: ZarrConfigManager) -> None: + # --- positive assertions: each distinct return shape --- + typing.assert_type(cfg.get("array.order"), typing.Literal["C", "F"]) + typing.assert_type(cfg.get("async.concurrency"), int) + typing.assert_type(cfg.get("array.write_empty_chunks"), bool) + typing.assert_type(cfg.get("async.timeout"), float | None) + typing.assert_type(cfg.get("threading.max_workers"), int | None) + typing.assert_type(cfg.get("default_zarr_format"), typing.Literal[2, 3]) + typing.assert_type(cfg.get("buffer"), str) + typing.assert_type(cfg.array.order, typing.Literal["C", "F"]) + + # --- negative: precision-from-above guards --- + # The return type is Literal["C","F"], which is narrower than str. + # If the overload were widened to -> str, assert_type would pass and + # the ignore below would become unused, causing warn_unused_ignores to + # fail CI. + typing.assert_type(cfg.get("array.order"), str) # type: ignore[assert-type] + typing.assert_type(cfg.get("default_zarr_format"), int) # type: ignore[assert-type] + + # --- negative: bad key type must be rejected by all overloads --- + cfg.get(123) # type: ignore[call-overload]