From 7841ce4b45b06d7de441888ead460b9551dec02f Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Mon, 4 May 2026 14:15:08 -0400 Subject: [PATCH 01/11] Fixed pickling error caused by `AsdfObject` inheritance --- asdf/tags/core/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/asdf/tags/core/__init__.py b/asdf/tags/core/__init__.py index 2b74bcb13..2f42c5fdf 100644 --- a/asdf/tags/core/__init__.py +++ b/asdf/tags/core/__init__.py @@ -24,7 +24,10 @@ # to pass an isinstance(..., dict) check and to allow it to be "lazy" # loaded when "lazy_tree=True". class AsdfObject(collections.UserDict, dict): - pass + def __reduce__(self): + # Necessary for correct pickling/unpickling + # Otherwise pickle will use dict's reduce method which causes UserDict to fail to unpickle + return super(collections.UserDict, self).__reduce__() class Software(dict): From c9cb35d305b4f6396848c1577a6c05de0eca81e1 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Mon, 4 May 2026 15:04:01 -0400 Subject: [PATCH 02/11] Fixed pickling error caused by local validator function --- asdf/extension/_manager.py | 87 +++++++++++++++++------------------- asdf/extension/_validator.py | 21 +++++++-- asdf/schema.py | 2 +- 3 files changed, 59 insertions(+), 51 deletions(-) diff --git a/asdf/extension/_manager.py b/asdf/extension/_manager.py index d02b35600..8d2b9c87e 100644 --- a/asdf/extension/_manager.py +++ b/asdf/extension/_manager.py @@ -1,11 +1,23 @@ +from __future__ import annotations + import sys +from dataclasses import dataclass from functools import lru_cache +from typing import TYPE_CHECKING from asdf.tagged import Tagged from asdf.util import get_class_name, uri_match from ._extension import ExtensionProxy +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Mapping + from typing import Any + + from asdf.exceptions import ValidationError + from asdf.extension import Validator + from asdf.typing import TreeKey + def _resolve_type(path): """ @@ -317,7 +329,7 @@ def _get_cached_extension_manager(extensions): class ValidatorManager: """ - Wraps a list of custom validators and indexes them by schema property. + Wraps a list of custom validators and binds them to their associated schemas. Parameters ---------- @@ -325,62 +337,43 @@ class ValidatorManager: List of validators to manage. """ - def __init__(self, validators): - self._validators = list(validators) + def __init__(self, validators: Iterable[Validator]): + by_schema_property = {} + for validator in validators: + if validator.schema_property not in by_schema_property: + by_schema_property[validator.schema_property] = set() - self._validators_by_schema_property = {} - for validator in self._validators: - if validator.schema_property not in self._validators_by_schema_property: - self._validators_by_schema_property[validator.schema_property] = set() - self._validators_by_schema_property[validator.schema_property].add(validator) + by_schema_property[validator.schema_property].add(validator) - self._jsonschema_validators_by_schema_property = {} - for schema_property in self._validators_by_schema_property: - self._jsonschema_validators_by_schema_property[schema_property] = self._get_jsonschema_validator( + self._validators = { + schema_property: BoundValidators( schema_property, + frozenset(validators), ) + for schema_property, validators in by_schema_property.items() + } - def validate(self, schema_property, schema_property_value, node, schema): - """ - Validate an ASDF tree node against custom validators for a schema property. + def validators(self) -> dict[str, BoundValidators]: + """Get a dictionary mapping schema names to callable validator functions.""" + return self._validators - Parameters - ---------- - schema_property : str - Name of the schema property (identifies the validator(s) to use). - schema_property_value : object - Value of the schema property. - node : asdf.tagged.Tagged - The ASDF node to validate. - schema : dict - The schema object that contains the property that triggered - the validation. - - Yields - ------ - asdf.exceptions.ValidationError - """ - if schema_property in self._validators_by_schema_property: - for validator in self._validators_by_schema_property[schema_property]: - if _validator_matches(validator, node): - yield from validator.validate(schema_property_value, node, schema) - def get_jsonschema_validators(self): - """ - Get a dictionary of validator methods suitable for use - with the jsonschema library. +@dataclass(frozen=True, slots=True) +class BoundValidators: + """Callable that wraps the `Validator.validate` methods of a set of `Validator` objects. - Returns - ------- - dict of str: callable - """ - return dict(self._jsonschema_validators_by_schema_property) + Each validator is always passed `schema_property` as its first argument regardless of the actual input schema. + """ - def _get_jsonschema_validator(self, schema_property): - def _validator(_, schema_property_value, node, schema): - return self.validate(schema_property, schema_property_value, node, schema) + schema_property: str + validators: frozenset[Validator] - return _validator + def __call__( + self, _schema_property: Any, schema_property_value: Any, node: Tagged, schema: Mapping[TreeKey, Any] + ) -> Iterator[ValidationError]: + for validator in self.validators: + if _validator_matches(validator, node): + yield from validator.validate(schema_property_value, node, schema) def _validator_matches(validator, node): diff --git a/asdf/extension/_validator.py b/asdf/extension/_validator.py index c695ac7a9..155e2d3b3 100644 --- a/asdf/extension/_validator.py +++ b/asdf/extension/_validator.py @@ -1,4 +1,14 @@ +from __future__ import annotations + import abc +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Mapping + + from asdf.exceptions import ValidationError + from asdf.tagged import Tagged + from asdf.typing import TreeKey class Validator(abc.ABC): @@ -8,13 +18,14 @@ class Validator(abc.ABC): """ @abc.abstractproperty - def schema_property(self): + def schema_property(self) -> str: """ Name of the schema property used to invoke this validator. """ + ... @abc.abstractproperty - def tags(self): + def tags(self) -> Iterable[str]: """ Get the YAML tags that are appropriate to this validator. URI patterns are permitted, see `asdf.util.uri_match` for details. @@ -24,9 +35,12 @@ def tags(self): iterable of str Tag URIs or URI patterns. """ + ... @abc.abstractmethod - def validate(self, schema_property_value, node, schema): + def validate( + self, schema_property_value: Any, node: Tagged, schema: Mapping[TreeKey, Any] + ) -> Iterator[ValidationError]: """ Validate the given node from the ASDF tree. @@ -54,3 +68,4 @@ def validate(self, schema_property_value, node, schema): asdf.exceptions.ValidationError Yield an instance of ValidationError for each error present in the node. """ + ... diff --git a/asdf/schema.py b/asdf/schema.py index 068ae9e9c..a7f6f67f2 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -552,7 +552,7 @@ def get_validator( if validators is None: validators = util.HashableDict(YAML_VALIDATORS.copy()) - validators.update(ctx._extension_manager.validator_manager.get_jsonschema_validators()) + validators.update(ctx._extension_manager.validator_manager.validators()) kwargs["resolver"] = _make_jsonschema_refresolver() From 38d4c5497c147713cb6b026465a8a8ec2fb1dbfd Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Mon, 4 May 2026 15:14:22 -0400 Subject: [PATCH 03/11] Added test case for pickling asdf file --- asdf/_tests/test_asdf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/asdf/_tests/test_asdf.py b/asdf/_tests/test_asdf.py index b7095467f..cf1cdaa3d 100644 --- a/asdf/_tests/test_asdf.py +++ b/asdf/_tests/test_asdf.py @@ -1,5 +1,7 @@ import os +import pickle +import numpy as np import pytest from asdf import config_context @@ -379,3 +381,12 @@ def test_fsspec_http(httpserver): with fsspec.open(fn) as f: af = open_asdf(f) assert_tree_match(tree, af.tree) + + +def test_asdf_file_pickle_from_dict(): + """Verify that an AsdfFile created from a dict (with no file descriptor) can be pickled""" + tree = {"a": 1, "b": {"c": 2, "d": np.ones((10, 10))}} + af = AsdfFile(tree) + pkl = pickle.dumps(af) + loaded = pickle.loads(pkl) # noqa: S301 + assert_tree_match(af.tree, loaded.tree) From 6599a470bcef6907154c7e5d68f041991efa66c3 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Mon, 4 May 2026 15:24:38 -0400 Subject: [PATCH 04/11] Added changelog entry --- changes/2038.bugfix.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/2038.bugfix.rst diff --git a/changes/2038.bugfix.rst b/changes/2038.bugfix.rst new file mode 100644 index 000000000..8cbb8fc80 --- /dev/null +++ b/changes/2038.bugfix.rst @@ -0,0 +1,3 @@ +`AsdfFile` instances can now be pickled as long as they don't reference any open file descriptors. +Currently that means that pickling is only supported for instances created from an in-memory dictionary with no blocks. +Additional ``pickle`` support will be added in a future release. From 2561d8a5dfe094d73128403e562eecc358a7acf0 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Mon, 4 May 2026 15:37:20 -0400 Subject: [PATCH 05/11] Re-enabled default coverage exclusions --- pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8086afb94..2f1fdab75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,9 +139,7 @@ omit = [ ] [tool.coverage.report] -exclude_lines = [ - # Have to re-enable the standard pragma - "pragma: no cover", +exclude_also = [ # Don't complain about packages we have installed "except ImportError", # Don't complain if tests don't hit assertions @@ -151,8 +149,6 @@ exclude_lines = [ 'def main\(.*\):', # Ignore branches that don't pertain to this version of Python "pragma: py{ ignore_python_version }", - # Ignore type-checking imports - "if TYPE_CHECKING:", ] [tool.ruff] From 903edc25a95db3151e74cbfebc9521a0102d5237 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Tue, 5 May 2026 09:24:45 -0400 Subject: [PATCH 06/11] Changed `ValidatorManager.validators` to a property --- asdf/extension/_manager.py | 3 ++- asdf/schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/asdf/extension/_manager.py b/asdf/extension/_manager.py index 8d2b9c87e..56b8a7db1 100644 --- a/asdf/extension/_manager.py +++ b/asdf/extension/_manager.py @@ -353,8 +353,9 @@ def __init__(self, validators: Iterable[Validator]): for schema_property, validators in by_schema_property.items() } + @property def validators(self) -> dict[str, BoundValidators]: - """Get a dictionary mapping schema names to callable validator functions.""" + """Dictionary mapping schema names to callable validator functions.""" return self._validators diff --git a/asdf/schema.py b/asdf/schema.py index a7f6f67f2..4da7d777c 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -552,7 +552,7 @@ def get_validator( if validators is None: validators = util.HashableDict(YAML_VALIDATORS.copy()) - validators.update(ctx._extension_manager.validator_manager.validators()) + validators.update(ctx._extension_manager.validator_manager.validators) kwargs["resolver"] = _make_jsonschema_refresolver() From fbac676e7e00a362ab8a1a5ad5fe5df246a6cf76 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Tue, 5 May 2026 15:26:17 -0400 Subject: [PATCH 07/11] Restored previous API for `ValidatorManager` --- asdf/extension/_manager.py | 29 ++++++++++++++++++++++++++++- asdf/schema.py | 2 +- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/asdf/extension/_manager.py b/asdf/extension/_manager.py index 56b8a7db1..1fc1f3ba9 100644 --- a/asdf/extension/_manager.py +++ b/asdf/extension/_manager.py @@ -354,10 +354,37 @@ def __init__(self, validators: Iterable[Validator]): } @property - def validators(self) -> dict[str, BoundValidators]: + def bound_validators(self) -> dict[str, BoundValidators]: """Dictionary mapping schema names to callable validator functions.""" return self._validators + def validate( + self, schema_property: str, schema_property_value: Any, node: Tagged, schema: Mapping[TreeKey, Any] + ) -> Iterator[ValidationError]: + """Validate an ASDF tree node against custom validators for a schema property. + + Parameters + ---------- + schema_property : str + Name of the schema property (identifies the validator(s) to use). + schema_property_value : object + Value of the schema property. + node : asdf.tagged.Tagged + The ASDF node to validate. + schema : dict + The schema object that contains the property that triggered + the validation. + + Yields + ------ + asdf.exceptions.ValidationError + """ + yield from self.bound_validators[schema_property](None, schema_property_value, node, schema) + + def get_jsonschema_validators(self) -> dict[str, BoundValidators]: + """Get a dictionary mapping schema names to callable validator functions.""" + return self._validators + @dataclass(frozen=True, slots=True) class BoundValidators: diff --git a/asdf/schema.py b/asdf/schema.py index 4da7d777c..1469992f4 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -552,7 +552,7 @@ def get_validator( if validators is None: validators = util.HashableDict(YAML_VALIDATORS.copy()) - validators.update(ctx._extension_manager.validator_manager.validators) + validators.update(ctx._extension_manager.validator_manager.bound_validators) kwargs["resolver"] = _make_jsonschema_refresolver() From 91f15f29364cb3373d3fec7cd9080ef1eebae154 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Wed, 13 May 2026 12:15:33 -0400 Subject: [PATCH 08/11] Marked `ValidatorManager.get_jsonschema_validators` as deprecated --- asdf/extension/_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/asdf/extension/_manager.py b/asdf/extension/_manager.py index 1fc1f3ba9..58c4cfaa2 100644 --- a/asdf/extension/_manager.py +++ b/asdf/extension/_manager.py @@ -5,6 +5,8 @@ from functools import lru_cache from typing import TYPE_CHECKING +from typing_extensions import deprecated + from asdf.tagged import Tagged from asdf.util import get_class_name, uri_match @@ -381,6 +383,7 @@ def validate( """ yield from self.bound_validators[schema_property](None, schema_property_value, node, schema) + @deprecated("use bound_validators instead") def get_jsonschema_validators(self) -> dict[str, BoundValidators]: """Get a dictionary mapping schema names to callable validator functions.""" return self._validators From 95135be628f124f0191471313acb588bb55c0094 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Wed, 13 May 2026 12:36:50 -0400 Subject: [PATCH 09/11] Reverted `ValidatorManager` API --- asdf/extension/_manager.py | 41 ++++++++++++++------------------------ asdf/schema.py | 2 +- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/asdf/extension/_manager.py b/asdf/extension/_manager.py index 58c4cfaa2..74e8ec2ed 100644 --- a/asdf/extension/_manager.py +++ b/asdf/extension/_manager.py @@ -5,8 +5,6 @@ from functools import lru_cache from typing import TYPE_CHECKING -from typing_extensions import deprecated - from asdf.tagged import Tagged from asdf.util import get_class_name, uri_match @@ -340,25 +338,12 @@ class ValidatorManager: """ def __init__(self, validators: Iterable[Validator]): - by_schema_property = {} + self._validators = {} for validator in validators: - if validator.schema_property not in by_schema_property: - by_schema_property[validator.schema_property] = set() + if validator.schema_property not in self._validators: + self._validators[validator.schema_property] = set() - by_schema_property[validator.schema_property].add(validator) - - self._validators = { - schema_property: BoundValidators( - schema_property, - frozenset(validators), - ) - for schema_property, validators in by_schema_property.items() - } - - @property - def bound_validators(self) -> dict[str, BoundValidators]: - """Dictionary mapping schema names to callable validator functions.""" - return self._validators + self._validators[validator.schema_property].add(validator) def validate( self, schema_property: str, schema_property_value: Any, node: Tagged, schema: Mapping[TreeKey, Any] @@ -381,17 +366,21 @@ def validate( ------ asdf.exceptions.ValidationError """ - yield from self.bound_validators[schema_property](None, schema_property_value, node, schema) + for validator in self._validators[schema_property]: + if _validator_matches(validator, node): + yield from validator.validate(schema_property_value, node, schema) - @deprecated("use bound_validators instead") - def get_jsonschema_validators(self) -> dict[str, BoundValidators]: - """Get a dictionary mapping schema names to callable validator functions.""" - return self._validators + def get_jsonschema_validators(self) -> dict[str, JsonSchemaValidators]: + """Get a dictionary mapping schema names to ``jsonschema``-compatible validator functions.""" + return { + schema_property: JsonSchemaValidators(schema_property, frozenset(validators)) + for schema_property, validators in self._validators.items() + } @dataclass(frozen=True, slots=True) -class BoundValidators: - """Callable that wraps the `Validator.validate` methods of a set of `Validator` objects. +class JsonSchemaValidators: + """Callable that wraps a set of `Validator` objects to make them compatible with `jsonschema`. Each validator is always passed `schema_property` as its first argument regardless of the actual input schema. """ diff --git a/asdf/schema.py b/asdf/schema.py index 1469992f4..068ae9e9c 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -552,7 +552,7 @@ def get_validator( if validators is None: validators = util.HashableDict(YAML_VALIDATORS.copy()) - validators.update(ctx._extension_manager.validator_manager.bound_validators) + validators.update(ctx._extension_manager.validator_manager.get_jsonschema_validators()) kwargs["resolver"] = _make_jsonschema_refresolver() From 55085c6e55cde4bae88c8421bc0c9e2cc9cdc552 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Wed, 13 May 2026 13:01:33 -0400 Subject: [PATCH 10/11] Added validator manager test --- asdf/_tests/test_extension.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/asdf/_tests/test_extension.py b/asdf/_tests/test_extension.py index de0c0c1fa..ae13a46ad 100644 --- a/asdf/_tests/test_extension.py +++ b/asdf/_tests/test_extension.py @@ -20,7 +20,8 @@ Validator, get_cached_extension_manager, ) -from asdf.extension._manager import _resolve_type +from asdf.extension._manager import ValidatorManager, _resolve_type +from asdf.tagged import TaggedList from asdf.testing.helpers import roundtrip_object @@ -734,6 +735,31 @@ def test_validator(): af.validate() +class ValidatorFailOn(Validator): + schema_property = "fail" + tags = ["fail"] + + def __init__(self, fail_on): + self.fail_on = fail_on + + def validate(self, schema_property_value, node, schema): + if schema_property_value == self.fail_on: + yield ValidationError("Node was doomed to fail") + + +def test_validator_manager(): + validator = ValidatorManager([ValidatorFailOn("bar")]) + errs = list(validator.validate("fail", "foo", TaggedList([], "fail"), {})) + assert len(errs) == 0 + + errs = list(validator.validate("fail", "bar", TaggedList([], "other"), {})) + assert len(errs) == 0 + + errs = list(validator.validate("fail", "bar", TaggedList([], "fail"), {})) + assert len(errs) == 1 + assert isinstance(errs[0], ValidationError) + + def test_converter_deferral(): class Bar: def __init__(self, value): From 4ecd798d5de2917680a7a0faff17269c0f2b9347 Mon Sep 17 00:00:00 2001 From: Sydney Duckworth Date: Mon, 18 May 2026 13:30:22 -0400 Subject: [PATCH 11/11] Removed changelog entry --- changes/2038.bugfix.rst | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 changes/2038.bugfix.rst diff --git a/changes/2038.bugfix.rst b/changes/2038.bugfix.rst deleted file mode 100644 index 8cbb8fc80..000000000 --- a/changes/2038.bugfix.rst +++ /dev/null @@ -1,3 +0,0 @@ -`AsdfFile` instances can now be pickled as long as they don't reference any open file descriptors. -Currently that means that pickling is only supported for instances created from an in-memory dictionary with no blocks. -Additional ``pickle`` support will be added in a future release.