Add support for ASDF serialisation and deserialisation

[Cadair]

This PR adds support for saving and loading the vast majority of ndcube objects to/from ASDF files.

Thanks to @ViciousEagle03 who did the majority of this work.

---

• Add support for Add serialization logic for uncertainty objects astropy/asdf-astropy#239
• Add support for Fits wcs astropy/asdf-astropy#246
• Add support for MultipleTableCoordinate
• Add documentation
• Bump all schemas & tags to 1.0
• Add support for NDMeta
• Test the few untested lines in converters
• Validate manually that having a reference to the parent cube in extra and global coords doesn’t cause trouble.
• Add support for HighLevelWCSWrapper in asdf-astropy

[Cadair]

This was a fun bug to discover. I ended up with a dict with ValueErrors as the values.

[DanRyanIrish]

In that case, this probably deserves a bugfix changelog file.

[DanRyanIrish]

Left a couple small comments that may require changes. Didn't look through the tests. Not an ASDF expert, but didn't see anything obvious that I didn't comment on.

[braingram]

A few comments on the schemas.

[Cadair]

From testing this with the dkist tools I think this needs to be

Suggested change

	- {}
	- type: object

@braingram ?

[braingram]

Would you share how {} didn't work? It's an empty schema (and valid for everything). type: object is more specific but perhaps that's fine.

[Cadair]

It threw me a fun error about there being no context. I think because the schema evaluated to false.

[braingram]

Would you share the error? {} is an empty schema and should be valid for anything.

[Cadair]

Here's the full traceback:

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File ~/DKIST/l2_test_writer.py:10
      6 # l2data.meta["fileuris"] = l2data["opticalDepth"].files.filenames
      8 af = asdf.AsdfFile(tree={"inversion": l2data})
---> 10 af.write_to("l2_test.asdf")
     12 # with asdf.open("l2_test_edit.asdf") as af:
     13 #     print(af["inversion"])

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_asdf.py:1246, in AsdfFile.write_to(self, fd, all_array_storage, all_array_compression, compression_kwargs, pad_blocks, include_block_index, version)
   1244 try:
   1245     with generic_io.get_file(fd, mode="w") as fd:
-> 1246         self._serial_write(fd, pad_blocks, include_block_index)
   1247 finally:
   1248     if version is not None:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_asdf.py:1013, in AsdfFile._serial_write(self, fd, pad_blocks, include_block_index)
   1010     if "history" in self._tree:
   1011         tree["history"] = copy.deepcopy(self._tree["history"])
-> 1013     self._write_tree(tree, fd, pad_blocks)
   1014     self._blocks.write(pad_blocks, include_block_index)
   1015 finally:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_asdf.py:985, in AsdfFile._write_tree(self, tree, fd, pad_blocks)
    982         else:
    983             tagged_tree.pop("history", None)
--> 985     yamlutil.dump_tree(
    986         tree,
    987         fd,
    988         self,
    989         tree_finalizer=_tree_finalizer,
    990         _serialization_context=serialization_context,
    991     )
    993 if pad_blocks:
    994     padding = util.calculate_padding(fd.tell(), pad_blocks, fd.block_size)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/yamlutil.py:418, in dump_tree(tree, fd, ctx, tree_finalizer, _serialization_context)
    416 if tree_finalizer is not None:
    417     tree_finalizer(tree)
--> 418 schema.validate(tree, ctx)
    420 # add yaml %TAG definitions from extensions
    421 if _serialization_context:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/schema.py:637, in validate(instance, ctx, schema, validators, reading, *args, **kwargs)
    634     ctx = AsdfFile()
    636 validator = get_validator({} if schema is None else schema, ctx, validators, None, *args, **kwargs)
--> 637 validator.validate(instance)
    639 additional_validators = [_validate_large_literals]
    640 if ctx.version >= versioning.RESTRICTED_KEYS_MIN_VERSION:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:311, in create.<locals>.Validator.validate(self, *args, **kwargs)
    310 def validate(self, *args, **kwargs):
--> 311     for error in self.iter_errors(*args, **kwargs):
    312         raise error

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/schema.py:302, in _create_validator.<locals>._patch_iter_errors.<locals>.iter_errors(self, instance, *args, **kwargs)
    300 if isinstance(instance, dict):
    301     for val in instance.values():
--> 302         yield from self.iter_errors(val)
    304 elif isinstance(instance, list):
    305     for val in instance:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/schema.py:296, in _create_validator.<locals>._patch_iter_errors.<locals>.iter_errors(self, instance, *args, **kwargs)
    294 try:
    295     with self.resolver.resolving(schema_uri) as resolved:
--> 296         yield from self.descend(instance, resolved)
    297 except RefResolutionError:
    298     warnings.warn(f"Unable to locate schema file for '{tag}': '{schema_uri}'", AsdfWarning)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:303, in create.<locals>.Validator.descend(self, instance, schema, path, schema_path)
    302 def descend(self, instance, schema, path=None, schema_path=None):
--> 303     for error in self.evolve(schema=schema).iter_errors(instance):
    304         if path is not None:
    305             error.path.appendleft(path)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/schema.py:308, in _create_validator.<locals>._patch_iter_errors.<locals>.iter_errors(self, instance, *args, **kwargs)
    306             yield from self.iter_errors(val)
    307 else:
--> 308     yield from original_iter_errors(self, instance)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:286, in create.<locals>.Validator.iter_errors(self, instance, _schema)
    283     continue
    285 errors = validator(self, v, instance, _schema) or ()
--> 286 for error in errors:
    287     # set details if not already set by the called fn
    288     error._set(
    289         validator=k,
    290         validator_value=v,
   (...)    293         type_checker=self.TYPE_CHECKER,
    294     )
    295     if k not in {"if", "$ref"}:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/_validators.py:332, in properties(validator, properties, instance, schema)
    330 for property, subschema in properties.items():
    331     if property in instance:
--> 332         yield from validator.descend(
    333             instance[property],
    334             subschema,
    335             path=property,
    336             schema_path=property,
    337         )

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:303, in create.<locals>.Validator.descend(self, instance, schema, path, schema_path)
    302 def descend(self, instance, schema, path=None, schema_path=None):
--> 303     for error in self.evolve(schema=schema).iter_errors(instance):
    304         if path is not None:
    305             error.path.appendleft(path)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/schema.py:308, in _create_validator.<locals>._patch_iter_errors.<locals>.iter_errors(self, instance, *args, **kwargs)
    306             yield from self.iter_errors(val)
    307 else:
--> 308     yield from original_iter_errors(self, instance)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:286, in create.<locals>.Validator.iter_errors(self, instance, _schema)
    283     continue
    285 errors = validator(self, v, instance, _schema) or ()
--> 286 for error in errors:
    287     # set details if not already set by the called fn
    288     error._set(
    289         validator=k,
    290         validator_value=v,
   (...)    293         type_checker=self.TYPE_CHECKER,
    294     )
    295     if k not in {"if", "$ref"}:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/_validators.py:25, in patternProperties(validator, patternProperties, instance, schema)
     23 for k, v in instance.items():
     24     if re.search(pattern, k):
---> 25         yield from validator.descend(
     26             v, subschema, path=k, schema_path=pattern,
     27         )

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:303, in create.<locals>.Validator.descend(self, instance, schema, path, schema_path)
    302 def descend(self, instance, schema, path=None, schema_path=None):
--> 303     for error in self.evolve(schema=schema).iter_errors(instance):
    304         if path is not None:
    305             error.path.appendleft(path)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/schema.py:308, in _create_validator.<locals>._patch_iter_errors.<locals>.iter_errors(self, instance, *args, **kwargs)
    306             yield from self.iter_errors(val)
    307 else:
--> 308     yield from original_iter_errors(self, instance)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:286, in create.<locals>.Validator.iter_errors(self, instance, _schema)
    283     continue
    285 errors = validator(self, v, instance, _schema) or ()
--> 286 for error in errors:
    287     # set details if not already set by the called fn
    288     error._set(
    289         validator=k,
    290         validator_value=v,
   (...)    293         type_checker=self.TYPE_CHECKER,
    294     )
    295     if k not in {"if", "$ref"}:

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/_validators.py:368, in anyOf(validator, anyOf, instance, schema)
    366 all_errors = []
    367 for index, subschema in enumerate(anyOf):
--> 368     errs = list(validator.descend(instance, subschema, schema_path=index))
    369     if not errs:
    370         break

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/_jsonschema/validators.py:303, in create.<locals>.Validator.descend(self, instance, schema, path, schema_path)
    302 def descend(self, instance, schema, path=None, schema_path=None):
--> 303     for error in self.evolve(schema=schema).iter_errors(instance):
    304         if path is not None:
    305             error.path.appendleft(path)

File ~/.virtualenvs/dkist-l2/lib/python3.13/site-packages/asdf/schema.py:288, in _create_validator.<locals>._patch_iter_errors.<locals>.iter_errors(self, instance, *args, **kwargs)
    286 if not self.schema:
    287     tag = getattr(instance, "_tag", None)
--> 288     if tag is not None and self.serialization_context.extension_manager.handles_tag_definition(tag):
    289         tag_def = self.serialization_context.extension_manager.get_tag_definition(tag)
    290         schema_uris = tag_def.schema_uris

AttributeError: 'NoneType' object has no attribute 'extension_manager'

[braingram]

Thanks. There is something odd going on there that doesn't look to be schema related. Is it possible to share the environment and files for me to try and reproduce that error?