Implement support for extension URIs

CHANGES.rst

@@ -16,6 +16,9 @@
 
 - Drop support for Python 3.5. [#856]
 
+- Add new extension API to support versioned extensions.
+  [#850]
+
 2.7.0 (2020-07-23)
 ------------------
 

asdf/asdf.py

@@ -69,8 +69,11 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None,
             automatically determined from the associated file object,
             if possible and if created from `AsdfFile.open`.
 
-        extensions : asdf.extension.AsdfExtension or asdf.extension.AsdfExtensionList or list of asdf.extension.AsdfExtension
+        extensions : object, optional
             Additional extensions to use when reading and writing the file.
+            May be any of the following: `asdf.extension.AsdfExtension`, `str`
+            extension URI, `asdf.extension.AsdfExtensionList` or a `list`
+            of URIs and/or extensions.
 
         version : str, optional
             The ASDF Standard version.  If not provided, defaults to the
@@ -247,60 +250,109 @@ def __exit__(self, type, value, traceback):
         self.close()
 
     def _check_extensions(self, tree, strict=False):
+        """
+        Compare the user's installed extensions to metadata in the tree
+        and warn when a) an extension is missing or b) an extension is
+        present but the file was written with a later version of the
+        extension's package.
+
+        Parameters
+        ----------
+        tree : AsdfObject
+            Fully converted tree of custom types.
+        strict : bool, optional
+            Set to `True` to convert warnings to exceptions.
+        """
         if 'history' not in tree or not isinstance(tree['history'], dict) or \
                 'extensions' not in tree['history']:
             return
 
         for extension in tree['history']['extensions']:
-            installed = next((e for e in self.extensions if e.class_name == extension.extension_class), None)
+            installed = None
+            for ext in self.extensions:
+                if (extension.extension_uri is not None and extension.extension_uri == ext.extension_uri
+                    or extension.extension_uri is None and extension.extension_class in ext.legacy_class_names):
+                    installed = ext
+                    break
 
             filename = "'{}' ".format(self._fname) if self._fname else ''
+            if extension.extension_uri is not None:
+                extension_description = "URI '{}'".format(extension.extension_uri)
+            else:
+                extension_description = "class '{}'".format(extension.extension_class)
+            if extension.software is not None:
+                extension_description += " (from package {}=={})".format(
+                    extension.software["name"],
+                    extension.software["version"],
+                )
+
             if installed is None:
-                msg = "File {}was created with extension '{}', which is " \
+                msg = (
+                    "File {}was created with extension {}, which is "
                     "not currently installed"
-                if extension.software:
-                    msg += " (from package {}-{})".format(
-                        extension.software['name'],
-                        extension.software['version'])
-                fmt_msg = msg.format(filename, extension.extension_class)
+                ).format(filename, extension_description)
                 if strict:
-                    raise RuntimeError(fmt_msg)
+                    raise RuntimeError(msg)
                 else:
-                    warnings.warn(fmt_msg, AsdfWarning)
+                    warnings.warn(msg, AsdfWarning)
 
             elif extension.software:
-                # Local extensions may not have a real version
-                if not installed.package_version:
+                # Local extensions may not have a real version.  If the package name changed,
+                # then the version sequence may have been reset.
+                if installed.package_version is None or installed.package_name != extension.software['name']:
                     continue
                 # Compare version in file metadata with installed version
                 if parse_version(installed.package_version) < parse_version(extension.software['version']):
-                    msg = "File {}was created with extension '{}' from " \
-                    "package {}-{}, but older version {}-{} is installed"
-                    fmt_msg = msg.format(
-                        filename, extension.extension_class,
-                        extension.software['name'],
-                        extension.software['version'],
-                        installed.package_name, installed.package_version)
+                    msg = (
+                        "File {}was created with extension {}, but older package ({}=={}) "
+                        "is installed."
+                    ).format(
+                        filename,
+                        extension_description,
+                        installed.package_name,
+                        installed.package_version,
+                    )
                     if strict:
-                        raise RuntimeError(fmt_msg)
+                        raise RuntimeError(msg)
                     else:
-                        warnings.warn(fmt_msg, AsdfWarning)
+                        warnings.warn(msg, AsdfWarning)
 
     def _process_extensions(self, requested_extensions):
+        """
+        Validate a list of extensions requested by the user and
+        add missing extensions registered with the current `AsdfConfig`.
+
+        Parameters
+        ----------
+        requested_extensions : object
+            May be any of the following: `asdf.extension.AsdfExtension`, `str`
+            extension URI, `asdf.extension.AsdfExtensionList` or a `list`
+            of URIs and/or extensions.
+
+        Returns
+        -------
+        list of asdf.extension.AsdfExtension
+        """
         if requested_extensions is None:
             requested_extensions = []
-        elif isinstance(requested_extensions, (AsdfExtension, ExtensionProxy)):
+        elif isinstance(requested_extensions, (AsdfExtension, ExtensionProxy, str)):
             requested_extensions = [requested_extensions]
         elif isinstance(requested_extensions, AsdfExtensionList):
             requested_extensions = requested_extensions.extensions
 
         if not isinstance(requested_extensions, list):
             raise TypeError(
-                "The extensions parameter must be an AsdfExtension, AsdfExtensionList, "
-                "or list of AsdfExtension."
+                "The extensions parameter must be an AsdfExtension, string URI, AsdfExtensionList, "
+                "or list of AsdfExtension/URI."
             )
 
-        requested_extensions = [ExtensionProxy.maybe_wrap(e) for e in requested_extensions]
+        def _get_extension(e):
+            if isinstance(e, str):
+                return get_config().get_extension(e)
+            else:
+                return ExtensionProxy.maybe_wrap(e)
+
+        requested_extensions = [_get_extension(e) for e in requested_extensions]
 
         extensions = []
         # Add requested extensions to the list first, so that they
@@ -312,6 +364,15 @@ def _process_extensions(self, requested_extensions):
         return extensions
 
     def _update_extension_history(self, serialization_context):
+        """
+        Update the extension metadata on this file's tree to reflect
+        extensions used during serialization.
+
+        Parameters
+        ----------
+        serialization_context : asdf.asdf.SerializationContext
+            The context that was used to serialize the tree.
+        """
         if serialization_context.version < versioning.NEW_HISTORY_FORMAT_MIN_VERSION:
             return
 
@@ -333,10 +394,13 @@ def _update_extension_history(self, serialization_context):
             ext_meta = ExtensionMetadata(extension_class=ext_name)
             if extension.package_name is not None:
                 ext_meta['software'] = Software(name=extension.package_name, version=extension.package_version)
+            if extension.extension_uri is not None:
+                ext_meta['extension_uri'] = extension.extension_uri
 
             for i, entry in enumerate(self.tree['history']['extensions']):
                 # Update metadata about this extension if it already exists
-                if entry.extension_class == ext_meta.extension_class:
+                if (entry.extension_uri is not None and entry.extension_uri == extension.extension_uri
+                    or entry.extension_class in extension.legacy_class_names):
                     self.tree['history']['extensions'][i] = ext_meta
                     break
             else:
@@ -1525,8 +1589,11 @@ def open_asdf(fd, uri=None, mode=None, validate_checksums=False,
         If `True`, validate the blocks against their checksums.
         Requires reading the entire file, so disabled by default.
 
-    extensions : asdf.extension.AsdfExtension or asdf.extension.AsdfExtensionList or list of asdf.extension.AsdfExtension
+    extensions : object, optional
         Additional extensions to use when reading and writing the file.
+        May be any of the following: `asdf.extension.AsdfExtension`, `str`
+        extension URI, `asdf.extension.AsdfExtensionList` or a `list`
+        of URIs and/or extensions.
 
     do_not_fill_defaults : bool, optional
         When `True`, do not fill in missing default values.

asdf/config.py

@@ -83,16 +83,26 @@ def remove_resource_mapping(self, mapping=None, *, package=None):
         mapping : collections.abc.Mapping, optional
             Mapping to remove.
         package : str, optional
-            Python package whose mappings will all be removed.
+            Remove only extensions provided by this package.  If the `mapping`
+            argument is omitted, then all mappings from this package will
+            be removed.
         """
-        with self._lock:
-            resource_mappings = self.resource_mappings
-            if mapping is not None:
+        if mapping is None and package is None:
+            raise ValueError("Must specify at least one of mapping or package")
+
+        if mapping is not None:
                 mapping = ResourceMappingProxy.maybe_wrap(mapping)
-                resource_mappings = [m for m in resource_mappings if m != mapping]
+
+        def _remove_condition(m):
+            result = True
+            if mapping is not None:
+                result = result and m == mapping
             if package is not None:
-                resource_mappings = [m for m in resource_mappings if m.package_name != package]
-            self._resource_mappings = resource_mappings
+                result = result and m.package_name == package
+            return result
+
+        with self._lock:
+            self._resource_mappings = [m for m in self.resource_mappings if not _remove_condition(m)]
             self._resource_manager = None
 
     def reset_resources(self):
@@ -154,19 +164,58 @@ def remove_extension(self, extension=None, *, package=None):
 
         Parameters
         ----------
-        extension : asdf.extension.AsdfExtension, optional
-            An extension instance to remove.
+        extension : asdf.extension.AsdfExtension or str, optional
+            An extension instance or URI to remove.
         package : str, optional
-            A Python package name whose extensions will all be removed.
+            Remove only extensions provided by this package.  If the `extension`
+            argument is omitted, then all extensions from this package will
+            be removed.
         """
-        with self._lock:
-            extensions = self.extensions
-            if extension is not None:
-                extension = ExtensionProxy.maybe_wrap(extension)
-                extensions = [e for e in extensions if e != extension]
+        if extension is None and package is None:
+            raise ValueError("Must specify at least one of extension or package")
+
+        if extension is not None and not isinstance(extension, str):
+            extension = ExtensionProxy.maybe_wrap(extension)
+
+        def _remove_condition(e):
+            result = True
+
+            if isinstance(extension, str):
+                result = result and e.extension_uri == extension
+            elif isinstance(extension, ExtensionProxy):
+                result = result and e == extension
+
             if package is not None:
-                extensions = [e for e in extensions if e.package_name != package]
-            self._extensions = extensions
+                result = result and e.package_name == package
+
+            return result
+
+        with self._lock:
+            self._extensions = [e for e in self.extensions if not _remove_condition(e)]
+
+    def get_extension(self, extension_uri):
+        """
+        Get an extension by URI.
+
+        Parameters
+        ----------
+        extension_uri : str
+            The URI of the extension to fetch.
+
+        Returns
+        -------
+        asdf.extension.AsdfExtension
+
+        Raises
+        ------
+        KeyError
+            If no such extension exists.
+        """
+        for extension in self.extensions:
+            if extension.extension_uri == extension_uri:
+                return extension
+
+        raise KeyError("No registered extension with URI '{}'".format(extension_uri))
 
     def reset_extensions(self):
         """

asdf/extension/_extension.py

@@ -27,6 +27,41 @@ def __init__(self, delegate, package_name=None, package_version=None):
 
         self._legacy = True
 
+        self._legacy_class_names = set()
+        for class_name in getattr(self._delegate, "legacy_class_names", []):
+            if isinstance(class_name, str):
+                self._legacy_class_names.add(class_name)
+            else:
+                raise TypeError("Extension property 'legacy_class_names' must contain str values")
+
+        if self._legacy:
+            self._legacy_class_names.add(self._class_name)
+
+    @property
+    def extension_uri(self):
+        """
+        Get the extension's identifying URI.
+
+        Returns
+        -------
+        str or None
+        """
+        return getattr(self._delegate, "extension_uri", None)
+
+    @property
+    def legacy_class_names(self):
+        """
+        Get the set of fully-qualified class names used by older
+        versions of this extension.  This allows a new-style
+        implementation of an extension to prevent warnings when a
+        legacy extension is missing.
+
+        Returns
+        -------
+        iterable of str
+        """
+        return self._legacy_class_names
+
     @property
     def types(self):
         """

asdf/fits_embed.py

@@ -196,10 +196,11 @@ def open(cls, fd, uri=None, validate_checksums=False, extensions=None,
             If `True`, validate the blocks against their checksums.
             Requires reading the entire file, so disabled by default.
 
-        extensions : list of AsdfExtension, optional
-            A list of extensions to the ASDF to support when reading
-            and writing ASDF files.  See `asdf.types.AsdfExtension` for
-            more information.
+        extensions : object, optional
+            Additional extensions to use when reading and writing the file.
+            May be any of the following: `asdf.extension.AsdfExtension`, `str`
+            extension URI, `asdf.extension.AsdfExtensionList` or a `list`
+            of URIs and/or extensions.
 
         ignore_version_mismatch : bool, optional
             When `True`, do not raise warnings for mismatched schema versions.

asdf/tags/core/__init__.py

@@ -45,6 +45,10 @@ class ExtensionMetadata(dict, AsdfType):
     name = 'core/extension_metadata'
     version = '1.0.0'
 
+    @property
+    def extension_uri(self):
+        return self.get('extension_uri')
+
     @property
     def extension_class(self):
         return self['extension_class']

asdf/tags/core/tests/test_history.py

@@ -159,7 +159,7 @@ def test_missing_extension_warning():
     """
 
     buff = yaml_to_asdf(yaml)
-    with pytest.warns(AsdfWarning, match="File was created with extension 'foo.bar.FooBar'"):
+    with pytest.warns(AsdfWarning, match="File was created with extension class 'foo.bar.FooBar'"):
         with asdf.open(buff):
             pass
 
@@ -177,7 +177,7 @@ def test_extension_version_warning():
     """
 
     buff = yaml_to_asdf(yaml)
-    with pytest.warns(AsdfWarning, match="File was created with extension 'asdf.extension.BuiltinExtension'"):
+    with pytest.warns(AsdfWarning, match="File was created with extension class 'asdf.extension.BuiltinExtension'"):
         with asdf.open(buff):
             pass