Merge pull request #192 from khaeru/enh/2024-W35

Improve handling of Metadata{StructureDefinition,Set} in SDMX-ML 2.1

doc/whatsnew.rst

@@ -3,8 +3,25 @@
 What's new?
 ***********
 
-.. Next release
-.. ============
+Next release
+============
+
+- :class:`MetadataStructureDefinition <.BaseMetadataStructureDefinition>` and :class:`MetadataSet <.BaseMetadataSet>` can be written to and read from SDMX-ML (:pull:`192`).
+
+  - Clarify differences between :attr:`.v21.MetadataSet.structured_by` and :attr:`.v30.MetadataSet.structured_by`, according to the respective standards documents.
+  - Read and write :class:`.MetadataAttribute`, :class:`.MetadataReport`, :class:`.ReportedAttribute`, :class:`.Period`, and associated classes and subclasses.
+  - :class:`.XHTMLAttributeValue` contents are stored as :mod:`lxml.etree` nodes.
+  - MetadataStructureDefinition is included when writing :class:`.StructureMessage`.
+
+- Update base url for :ref:`WB_WDI` source to use HTTPS instead of plain HTTP (:issue:`191`, :pull:`192`).
+- Improvements to :mod:`.reader.xml` and :mod:`.reader.xml.v21` (:pull:`192`).
+
+  - Correctly associate :class:`.Item` in :class:`.ItemScheme` with its parent, even if the parent is defined after the child (“forward reference”).
+  - Bugfix: correctly handle a :class:`.MaintainableArtefact` that is explicitly included in a message (that is, not merely referenced), but with :py:`is_external_reference = True`; the value given in the file is preserved.
+  - Bugfix: :class:`.FacetValueType` is written in UpperCamelCase per the standard.
+    The standard specifies lowerCamelCase only in the Information Model.
+  - Bugfix: erroneous extra :xml:`<Ref ... style="Ref"/>` attribute is no longer written.
+- Expand logged information in :meth:`.ComponentList.compare` (:pull:`192`).
 
 v2.16.0 (2024-08-16)
 ====================

pyproject.toml

@@ -50,9 +50,9 @@ tests = [
 ]
 
 [project.urls]
-homepage = "https://github.com/khaeru/sdmx"
-repository = "https://github.com/khaeru/sdmx"
-documentation = "https://sdmx1.readthedocs.io/en/latest"
+Homepage = "https://github.com/khaeru/sdmx"
+Repository = "https://github.com/khaeru/sdmx"
+Documentation = "https://sdmx1.readthedocs.io/en/latest"
 
 [tool.coverage.run]
 omit = [

sdmx/dictlike.py

@@ -98,11 +98,12 @@ def compare(self, other, strict=True):
             Passed to :func:`compare` for the values.
         """
         if set(self.keys()) != set(other.keys()):
-            log.info(f"Not identical: {sorted(self.keys())} / {sorted(other.keys())}")
+            log.debug(f"Not identical: {sorted(self.keys())} / {sorted(other.keys())}")
             return False
 
         for key, value in self.items():
             if not value.compare(other[key], strict):
+                log.debug(f"Not identical: {value} / {other[key]}")
                 return False
 
         return True

sdmx/format/xml/v21.py

@@ -25,6 +25,8 @@
         ("model.TargetReportPeriod", "md:ReportPeriod"),
         ("model.MetadataReport", ":Report"),
         ("model.MetadataReport", "md:Report"),
+        ("model.StartPeriod", "com:StartPeriod"),
+        ("model.EndPeriod", "com:EndPeriod"),
     ]
     + [
         (f"model.{name}", f"str:{name}")

sdmx/model/common.py

@@ -287,7 +287,8 @@ def compare(self, other, strict=True):
         return (
             compare("id", self, other, strict)
             and compare("uri", self, other, strict)
-            and compare("urn", self, other, strict)
+            # Allow non-strict comparison if self.urn is None
+            and compare("urn", self, other, strict and self.urn is not None)
         )
 
     def __hash__(self):
@@ -997,9 +998,14 @@ def compare(self, other, strict=True):
         strict : bool, optional
             Passed to :func:`.compare` and :meth:`.IdentifiableArtefact.compare`.
         """
-        return super().compare(other, strict) and all(
-            c.compare(other.get(c.id), strict) for c in self.components
-        )
+        result = True
+        for c in self.components:
+            try:
+                result &= c.compare(other.get(c.id), strict)
+            except KeyError:
+                log.debug(f"{other} has no component with ID {c.id!r}")
+                result = False
+        return result and super().compare(other, strict)
 
 
 # §4.5: Category Scheme
@@ -2151,6 +2157,13 @@ class BaseMetadataSet:
     publication_period: Optional[date] = None
     publication_year: Optional[date] = None
 
+    described_by: Optional[BaseMetadataflow] = None
+
+    #: Note that the class of this attribute differs from SDMX 2.1 to SDMX 3.0.
+    #: Compare :attr:`.v21.MetadataSet.structured_by` and
+    #: :attr:`.v30.MetadataSet.structured_by`.
+    structured_by: Optional[IdentifiableArtefact] = None
+
 
 # SDMX 2.1 §8: Hierarchical Code List
 # SDMX 3.0 §8: Hierarchy
@@ -2570,6 +2583,7 @@ class BaseContentConstraint:
         "HierarchicalCodelist",  # SDMX 2.1
         "Hierarchy",
         "Level",
+        "ValueList",  # SDMX 3.0
     },
     "conceptscheme": {"Concept", "ConceptScheme"},
     "datastructure": {

sdmx/model/v21.py

@@ -19,6 +19,7 @@
 )
 
 from sdmx.dictlike import DictLikeDescriptor
+from sdmx.util import compare
 
 from . import common
 from .common import (
@@ -344,9 +345,29 @@ class DimensionDescriptorValuesTarget(TargetObject):
     """SDMX 2.1 DimensionDescriptorValuesTarget."""
 
 
+@dataclass
 class IdentifiableObjectTarget(TargetObject):
     """SDMX 2.1 IdentifiableObjectTarget."""
 
+    #: Type of :class:`.IdentifiableArtefact` that is targeted.
+    object_type: Optional[Type[IdentifiableArtefact]] = None
+
+    def compare(self, other, strict=True):
+        """Return :obj:`True` if `self` is the same as `other`.
+
+        Two IdentifiableObjectTargets are the same if
+        :meth:`.IdentifiableArtefact.compare` is :obj:`True` and they have the same
+        :attr:`object_type`.
+
+        Parameters
+        ----------
+        strict : bool, optional
+            Passed to :func:`.compare`.
+        """
+        return super().compare(other, strict) and compare(
+            "object_type", self, other, strict
+        )
+
 
 class ReportPeriodTarget(TargetObject):
     """SDMX 2.1 ReportPeriodTarget."""
@@ -455,10 +476,11 @@ class NonEnumeratedAttributeValue(ReportedAttribute):
     """SDMX 2.1 NonEnumeratedAttributeValue."""
 
 
+@dataclass
 class OtherNonEnumeratedAttributeValue(NonEnumeratedAttributeValue):
     """SDMX 2.1 OtherNonEnumeratedAttributeValue."""
 
-    value: str
+    value: Optional[str] = None
 
 
 class TextAttributeValue(NonEnumeratedAttributeValue, common.BaseTextAttributeValue):
@@ -491,6 +513,9 @@ class MetadataSet(NameableArtefact, common.BaseMetadataSet):
 
     described_by: Optional[MetadataflowDefinition] = None
     # described_by: Optional[ReportStructure] = None
+
+    #: .. seealso::
+    #:    :attr:`.v30.MetadataSet.structured_by`, which has different semantics.
     structured_by: Optional[MetadataStructureDefinition] = None
 
     #: Analogous to :attr:`.v30.MetadataSet.provided_by`.

sdmx/model/v30.py

@@ -486,10 +486,20 @@ class MetadataSet(MaintainableArtefact, common.BaseMetadataSet):
     valid_to: Optional[str] = None
     set_id: Optional[str] = None
 
+    #: .. note::
+    #:    According to the standard, MetadataSet has **two** associations, both named
+    #:    :py:`.described_by`: one to a :class:`.Metadataflow`, and the other to a
+    #:    :class:`.MetadataProvisionAgreement`. :mod:`sdmx` implements the first,
+    #:    because it is consistent with SDMX 2.1.
     described_by: Optional[Metadataflow] = None
 
     # described_by: Optional[MetadataProvisionAgreement] = None
 
+    #: .. note::
+    #:    According to the standard, this differs from
+    #:    :attr:`v21.MetadataSet.structured_by` in that it points directly to
+    #:    :attr:`.MetadataStructureDefinition.attributes`, rather than to the
+    #:    MetadataStructureDefinition that contains the attribute descriptor.
     structured_by: Optional[MetadataAttributeDescriptor] = None
 
     #: Analogous to :attr:`.v21.MetadataSet.published_by`.

sdmx/reader/xml/common.py

@@ -545,7 +545,6 @@ def maintainable(self, cls, elem, **kwargs):
         the is_external_reference attribute set to :obj:`True`. Subsequent calls with
         the same object ID will return references to the same object.
         """
-        kwargs.setdefault("is_external_reference", elem is None)
         setdefault_attrib(
             kwargs,
             elem,
@@ -555,6 +554,12 @@ def maintainable(self, cls, elem, **kwargs):
             "validTo",
             "version",
         )
+
+        # Ensure is_external_reference and is_final are bool
+        try:
+            kwargs["is_external_reference"] = kwargs["is_external_reference"] == "true"
+        except (KeyError, SyntaxError):
+            kwargs.setdefault("is_external_reference", elem is None)
         kwargs["is_final"] = kwargs.get("is_final", None) == "true"
 
         # Create a candidate object
@@ -579,19 +584,13 @@ def maintainable(self, cls, elem, **kwargs):
         existing = self.get_single(cls, obj.id, version=obj.version)
 
         if existing and (
-            existing.compare(obj, strict=True) or existing.urn == sdmx.urn.make(obj)
+            existing.compare(obj, strict=True)
+            or (existing.urn or sdmx.urn.make(existing)) == sdmx.urn.make(obj)
         ):
             if elem is not None:
-                # Previously an external reference, now concrete
-                existing.is_external_reference = False
-
                 # Update `existing` from `obj` to preserve references
                 # If `existing` was a forward reference <Ref/>, its URN was not stored.
                 for attr in list(kwargs.keys()) + ["urn"]:
-                    # log.info(
-                    #     f"Updating {attr} {getattr(existing, attr)} "
-                    #     f"{getattr(obj, attr)}"
-                    # )
                     setattr(existing, attr, getattr(obj, attr))
 
             # Discard the candidate
@@ -616,14 +615,15 @@ def matching_class(cls):
 
 
 def setdefault_attrib(target, elem, *names):
+    """Update `target` from :py:`elem.attrib` for the given `names`."""
     try:
         for name in names:
             try:
                 target.setdefault(to_snake(name), elem.attrib[name])
             except KeyError:
                 pass
     except AttributeError:
-        pass
+        pass  # No elem.attrib; elem is None
 
 
 TO_SNAKE_RE = re.compile("([A-Z]+)")