From 3902d0f108303c7e42d18ff03cc2e4e090d9f25c Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:16:34 +0200
Subject: [PATCH 01/45] Add com:{Start,End}Period to .format.xml.v21

---
 sdmx/format/xml/v21.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sdmx/format/xml/v21.py b/sdmx/format/xml/v21.py
index bee7ff794..7b976e538 100644
--- a/sdmx/format/xml/v21.py
+++ b/sdmx/format/xml/v21.py
@@ -25,6 +25,8 @@
         ("model.TargetReportPeriod", "md:ReportPeriod"),
         ("model.MetadataReport", ":Report"),
         ("model.MetadataReport", "md:Report"),
+        ("model.StartPeriod", "com:StartPeriod"),
+        ("model.EndPeriod", "com:EndPeriod"),
     ]
     + [
         (f"model.{name}", f"str:{name}")

From 567ab64025d8ec99d8de09a15dfc1378ae4e26c5 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:19:06 +0200
Subject: [PATCH 02/45] Support .model.v30.ValueList in get_class()

---
 sdmx/model/common.py     |  1 +
 sdmx/tests/test_model.py | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/sdmx/model/common.py b/sdmx/model/common.py
index 55d822890..3a0566440 100644
--- a/sdmx/model/common.py
+++ b/sdmx/model/common.py
@@ -2570,6 +2570,7 @@ class BaseContentConstraint:
         "HierarchicalCodelist",  # SDMX 2.1
         "Hierarchy",
         "Level",
+        "ValueList",  # SDMX 3.0
     },
     "conceptscheme": {"Concept", "ConceptScheme"},
     "datastructure": {
diff --git a/sdmx/tests/test_model.py b/sdmx/tests/test_model.py
index 95d9dae1c..fc1817035 100644
--- a/sdmx/tests/test_model.py
+++ b/sdmx/tests/test_model.py
@@ -225,10 +225,20 @@ def test_complete(module, extra):
         ),
     ],
 )
-def test_get_class(args, expected):
+def test_get_class_v21(args, expected) -> None:
     assert expected is model.v21.get_class(**args)
 
 
+@pytest.mark.parametrize("args, expected", ((dict(name="ValueList"), v30.ValueList),))
+def test_get_class_v30(args, expected) -> None:
+    assert expected is model.v30.get_class(**args)
+
+
+@pytest.mark.parametrize("klass, expected", (("ValueList", "codelist"),))
+def test_package(klass, expected) -> None:
+    assert expected == model.v30.PACKAGE[klass]
+
+
 def test_deprecated_import():
     """Deprecation warning when importing SDMX 2.1-specific class from :mod:`.model`."""
     with pytest.warns(

From 182cb37887f39e3ab19b7ecc05daee85bb9281a2 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:22:28 +0200
Subject: [PATCH 03/45] Simplify construction of elements in .writer.xml

- Pass multiple child elements as positional args to Element(), instead
  of elem.append() or elem.extend().
- Return created elements immediately without a local variable.
---
 sdmx/writer/xml.py | 40 ++++++++++++++++++----------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 10bd43bbd..0a3acfc6a 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -400,16 +400,15 @@ def _concept(obj: model.Concept, **kwargs):
 
 @writer
 def _contact(obj: model.Contact):
-    elem = Element("str:Contact")
-    elem.extend(
-        i11lstring(obj.name, "com:Name")
-        + i11lstring(obj.org_unit, "str:Department")
-        + i11lstring(obj.responsibility, "str:Role")
-        + ([Element("str:Telephone", obj.telephone)] if obj.telephone else [])
-        + [Element("str:URI", value) for value in obj.uri]
-        + [Element("str:Email", value) for value in obj.email]
+    return Element(
+        "str:Contact",
+        *i11lstring(obj.name, "com:Name"),
+        *i11lstring(obj.org_unit, "str:Department"),
+        *i11lstring(obj.responsibility, "str:Role"),
+        *([Element("str:Telephone", obj.telephone)] if obj.telephone else []),
+        *[Element("str:URI", value) for value in obj.uri],
+        *[Element("str:Email", value) for value in obj.email],
     )
-    return elem
 
 
 # §3.3: Basic Inheritance
@@ -449,9 +448,7 @@ def _component(obj: model.Component, dsd):
 
 @writer
 def _cl(obj: model.ComponentList, *args):
-    elem = identifiable(obj)
-    elem.extend(writer.recurse(c, *args) for c in obj.components)
-    return elem
+    return identifiable(obj, *[writer.recurse(c, *args) for c in obj.components])
 
 
 # §4.5: CategoryScheme
@@ -459,14 +456,11 @@ def _cl(obj: model.ComponentList, *args):
 
 @writer
 def _cat(obj: model.Categorisation):
-    elem = maintainable(obj)
-    elem.extend(
-        [
-            reference(obj.artefact, tag="str:Source", style="Ref"),
-            reference(obj.category, tag="str:Target", style="Ref"),
-        ]
+    return maintainable(
+        obj,
+        reference(obj.artefact, tag="str:Source", style="Ref"),
+        reference(obj.category, tag="str:Target", style="Ref"),
     )
-    return elem
 
 
 # §10.3: Constraints
@@ -506,9 +500,11 @@ def _ms(obj: model.MemberSelection):
 
 @writer
 def _cr(obj: model.CubeRegion):
-    elem = Element("str:CubeRegion", include=str(obj.included).lower())
-    elem.extend(writer.recurse(ms) for ms in obj.member.values())
-    return elem
+    return Element(
+        "str:CubeRegion",
+        *[writer.recurse(ms) for ms in obj.member.values()],
+        include=str(obj.included).lower(),
+    )
 
 
 @writer

From 74aa411c507f9c5055d58a56dd496337dda12e7c Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:23:21 +0200
Subject: [PATCH 04/45] Raise informative exception from
 .writer.xml.annotable()

---
 sdmx/writer/xml.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 0a3acfc6a..48d0f97c0 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -284,6 +284,8 @@ def _a(obj: model.Annotation):
 def annotable(obj, *args, **kwargs) -> etree._Element:
     # Determine tag
     tag = kwargs.pop("_tag", tag_for_class(obj.__class__))
+    if tag is None:
+        raise NotImplementedError(f"Write {obj.__class__} to SDMX-ML")
 
     # Write Annotations
     e_anno = Element("com:Annotations", *[writer.recurse(a) for a in obj.annotations])

From 69ff0d391ab41734ba660a5a50fcc421c0d6d407 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:26:57 +0200
Subject: [PATCH 05/45] Write .model.v21.RangePeriod

- Write distinct tags for .v21.{Start,End}Period.
- Simplify writing DataKeySet and MemberSelection.
- Avoid use of typing.cast()
---
 sdmx/writer/xml.py | 39 +++++++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 48d0f97c0..c0aec8918 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -6,7 +6,7 @@
 # - writer functions for sdmx.model classes, in the same order as model.py
 
 import logging
-from typing import Iterable, List, Literal, cast
+from typing import Iterable, List, Literal
 
 from lxml import etree
 from lxml.builder import ElementMaker
@@ -479,9 +479,32 @@ def _dk(obj: model.DataKey):
 
 @writer
 def _dks(obj: model.DataKeySet):
-    elem = Element("str:DataKeySet", isIncluded=str(obj.included).lower())
-    elem.extend(writer.recurse(dk) for dk in obj.keys)
-    return elem
+    return Element(
+        "str:DataKeySet",
+        *[writer.recurse(dk) for dk in obj.keys],
+        isIncluded=str(obj.included).lower(),
+    )
+
+
+@writer
+def _mv(obj: model.MemberValue):
+    return Element("com:Value", obj.value)
+
+
+@writer
+def _rp(obj: model.RangePeriod):
+    return Element("com:TimeRange", writer.recurse(obj.start), writer.recurse(obj.end))
+
+
+@writer
+def _period(obj: common.Period):
+    """Includes :class:`.v21.StartPeriod` and :class:`.v21.EndPeriod`."""
+    return Element(
+        f"com:{obj.__class__.__name__}",
+        # `period` attribute as text
+        obj.period.isoformat(),
+        isInclusive=str(obj.is_inclusive).lower(),
+    )
 
 
 @writer
@@ -491,13 +514,9 @@ def _ms(obj: model.MemberSelection):
         model.DataAttribute: "Attribute",
     }[type(obj.values_for)]
 
-    elem = Element(f"com:{tag}", id=obj.values_for.id)
-    elem.extend(
-        # cast(): as of PR#30, only MemberValue is supported here
-        Element("com:Value", cast(model.MemberValue, mv).value)
-        for mv in obj.values
+    return Element(
+        f"com:{tag}", *[writer.recurse(v) for v in obj.values], id=obj.values_for.id
     )
-    return elem
 
 
 @writer

From dd260ed333371ac7151cd57ed3f2e3142c2e88ed Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:27:36 +0200
Subject: [PATCH 06/45] Add ESTAT/esms{,-structure}.xml to specimens

---
 sdmx/testing/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sdmx/testing/__init__.py b/sdmx/testing/__init__.py
index 9208e60da..773b9ad78 100644
--- a/sdmx/testing/__init__.py
+++ b/sdmx/testing/__init__.py
@@ -260,6 +260,7 @@ def __init__(self, base_path):
             for parts in [
                 ("INSEE", "CNA-2010-CONSO-SI-A17.xml"),
                 ("INSEE", "IPI-2010-A21.xml"),
+                ("ESTAT", "esms.xml"),
                 ("ESTAT", "footer.xml"),
                 ("ESTAT", "NAMA_10_GDP-ss.xml"),
             ]
@@ -274,6 +275,7 @@ def __init__(self, base_path):
                 ("ECB", "orgscheme.xml"),
                 ("ECB", "structureset-0.xml"),
                 ("ESTAT", "apro_mk_cola-structure.xml"),
+                ("ESTAT", "esms-structure.xml"),
                 ("ESTAT", "GOV_10Q_GGNFA.xml"),
                 ("ESTAT", "HCL_WSTATUS_SCL_BNSPART.xml"),
                 ("ESTAT", "HCL_WSTATUS_SCL_WSTATUSPR.xml"),

From 80ce4eceb1aa17554d82f2eeddee181d06a68851 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:28:10 +0200
Subject: [PATCH 07/45] Test round-trip of MetadataSet from/to XML

---
 sdmx/tests/writer/test_writer_xml.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sdmx/tests/writer/test_writer_xml.py b/sdmx/tests/writer/test_writer_xml.py
index 5860e7c6a..f232ae521 100644
--- a/sdmx/tests/writer/test_writer_xml.py
+++ b/sdmx/tests/writer/test_writer_xml.py
@@ -216,6 +216,7 @@ def test_ErrorMessage(errormessage):
         ("ECB_EXR/1/M.USD.EUR.SP00.A.xml", "ECB_EXR/1/structure.xml"),
         ("ECB_EXR/ng-ts.xml", "ECB_EXR/ng-structure-full.xml"),
         ("ECB_EXR/ng-ts-ss.xml", "ECB_EXR/ng-structure-full.xml"),
+        ("ESTAT/esms.xml", "ESTAT/esms-structure.xml"),
         # DSD reference does not round-trip correctly
         pytest.param(
             "ECB_EXR/rg-xs.xml",

From 014494658d7f6dc4da369c3492ebcfb3c68eb122 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:28:45 +0200
Subject: [PATCH 08/45] Test round-trip of RangePeriod from/to XML

---
 sdmx/tests/writer/test_writer_xml.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sdmx/tests/writer/test_writer_xml.py b/sdmx/tests/writer/test_writer_xml.py
index f232ae521..8cb24d41a 100644
--- a/sdmx/tests/writer/test_writer_xml.py
+++ b/sdmx/tests/writer/test_writer_xml.py
@@ -268,6 +268,7 @@ def test_data_roundtrip(pytestconfig, specimen, data_id, structure_id, tmp_path)
         ("INSEE/CNA-2010-CONSO-SI-A17-structure.xml", False),
         ("INSEE/IPI-2010-A21-structure.xml", False),
         ("INSEE/dataflow.xml", False),
+        ("OECD/actualconstraint-0.xml", True),
         ("SGR/common-structure.xml", True),
         ("UNSD/codelist_partial.xml", True),
         ("TEST/gh-149.xml", False),

From dfdbb091e90b37bde5f240d31a8b5590a8e61c81 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:29:17 +0200
Subject: [PATCH 09/45] Make OtherNonEnumeratedAttributeValue a dataclass

---
 sdmx/model/v21.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sdmx/model/v21.py b/sdmx/model/v21.py
index eedd48333..5b57a1cf9 100644
--- a/sdmx/model/v21.py
+++ b/sdmx/model/v21.py
@@ -455,10 +455,11 @@ class NonEnumeratedAttributeValue(ReportedAttribute):
     """SDMX 2.1 NonEnumeratedAttributeValue."""
 
 
+@dataclass
 class OtherNonEnumeratedAttributeValue(NonEnumeratedAttributeValue):
     """SDMX 2.1 OtherNonEnumeratedAttributeValue."""
 
-    value: str
+    value: Optional[str] = None
 
 
 class TextAttributeValue(NonEnumeratedAttributeValue, common.BaseTextAttributeValue):

From d097346e097b25890c8ec671f3c05974323b9e3f Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:31:16 +0200
Subject: [PATCH 10/45] Simplify reading of com:TimeRange et al.

---
 sdmx/reader/xml/v21.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index 3e0b2b90a..fb88d756f 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -765,19 +765,14 @@ def _dks(reader, elem):
 
 @end("com:StartPeriod com:EndPeriod")
 def _p(reader, elem):
-    # Store by element tag name
-    reader.push(
-        elem,
-        model.Period(
-            is_inclusive=elem.attrib["isInclusive"], period=isoparse(elem.text)
-        ),
-    )
+    cls = reader.class_for_tag(elem.tag)
+    return cls(is_inclusive=elem.attrib["isInclusive"], period=isoparse(elem.text))
 
 
 @end("com:TimeRange")
 def _tr(reader, elem):
-    return model.RangePeriod(
-        start=reader.pop_single("StartPeriod"), end=reader.pop_single("EndPeriod")
+    return v21.RangePeriod(
+        start=reader.pop_single(v21.StartPeriod), end=reader.pop_single(v21.EndPeriod)
     )
 
 

From 963ad87e68015bc2fbacac0d26260df39055a12a Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:36:57 +0200
Subject: [PATCH 11/45] Use correct root XML tag for MetadataMessage

---
 sdmx/writer/xml.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index c0aec8918..d4e37e596 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -119,12 +119,18 @@ def reference(obj, parent=None, tag=None, *, style: RefStyle):
 
 @writer
 def _dm(obj: message.DataMessage):
-    struct_spec = len(obj.data) and isinstance(
+    """DataMessage, including MetadataMessage."""
+    # Identify root tag
+    if len(obj.data) and isinstance(
         obj.data[0],
         (model.StructureSpecificDataSet, model.StructureSpecificTimeSeriesDataSet),
-    )
+    ):
+        tag = "mes:StructureSpecificData"
+    else:
+        tag = tag_for_class(type(obj))
 
-    elem = Element("mes:StructureSpecificData" if struct_spec else "mes:GenericData")
+    # Create the root element
+    elem = Element(tag)
 
     header = writer.recurse(obj.header)
     elem.append(header)

From 3f7d56ed7b477d704e5d73ff39de603127ff1a41 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:39:23 +0200
Subject: [PATCH 12/45] Write .v21.MetadataSet and contents to XML

---
 sdmx/writer/xml.py | 92 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 91 insertions(+), 1 deletion(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index d4e37e596..d89197663 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -6,7 +6,7 @@
 # - writer functions for sdmx.model classes, in the same order as model.py
 
 import logging
-from typing import Iterable, List, Literal
+from typing import Iterable, List, Literal, MutableMapping
 
 from lxml import etree
 from lxml.builder import ElementMaker
@@ -740,3 +740,93 @@ def _ds(obj: model.DataSet):
         elem.append(writer.recurse(obs, struct_spec=struct_spec))
 
     return elem
+
+
+# SDMX 2.1 §7.4: Metadata Set
+
+
+@writer
+def _mds(obj: model.MetadataSet):
+    attrib = {}
+    if obj.structured_by:
+        attrib["structureRef"] = obj.structured_by.id
+    return Element(
+        "mes:MetadataSet", *[writer.recurse(mdr) for mdr in obj.report], **attrib
+    )
+
+
+@writer
+def _mdr(obj: model.MetadataReport):
+    # TODO Write the id=… attribute
+    elem = Element("md:Report")
+
+    if obj.target:
+        elem.append(writer.recurse(obj.target))
+    if obj.attaches_to:
+        elem.append(writer.recurse(obj.attaches_to))
+
+    elem.append(
+        Element("md:AttributeSet", *[writer.recurse(ra) for ra in obj.metadata])
+    )
+
+    return elem
+
+
+@writer
+def _tok(obj: model.TargetObjectKey):
+    # TODO Write the id=… attribute
+    return Element(
+        "md:Target", *[writer.recurse(tov) for tov in obj.key_values.values()]
+    )
+
+
+@writer
+def _tov(obj: model.TargetObjectValue):
+    if isinstance(obj.value_for, str):
+        id_: str = obj.value_for
+    else:
+        id_ = obj.value_for.id
+
+    elem = Element("md:ReferenceValue", id=id_)
+
+    if isinstance(obj, model.TargetReportPeriod):
+        elem.append(Element("md:ReportPeriod", obj.report_period))
+    elif isinstance(obj, model.TargetIdentifiableObject):
+        elem.append(
+            Element("md:ObjectReference", Element("URN", sdmx.urn.make(obj.obj)))
+        )
+    else:
+        assert False
+
+    return elem
+
+
+@writer
+def _ra(obj: model.ReportedAttribute):
+    child = []
+    attrib: MutableMapping[str, str] = dict()
+
+    if isinstance(obj.value_for, str):
+        # NB value_for should be MetadataAttribute, but currently not due to limitations
+        #    of .reader.xml.v21
+        attrib.update(id=obj.value_for)
+    else:
+        attrib.update(id=obj.value_for.id)
+
+    if isinstance(obj, model.OtherNonEnumeratedAttributeValue):
+        # Only write the "value" attribute if defined; some attributes are only
+        # containers for child attributes
+        if obj.value:
+            attrib.update(value=obj.value)
+    elif isinstance(obj, model.XHTMLAttributeValue):
+        child.append(Element("com:StructuredText", obj.value))
+    else:
+        raise NotImplementedError
+
+    if len(obj.child):
+        # Add child ReportedAttribute within an AttributeSet
+        child.append(
+            Element("md:AttributeSet", *[writer.recurse(ra) for ra in obj.child])
+        )
+
+    return Element("md:ReportedAttribute", *child, **attrib)

From a83efe65650617d07ecf2747cfeecafaf4db1b33 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:41:00 +0200
Subject: [PATCH 13/45] Streamline some .reader.xml.v21 functions

---
 sdmx/reader/xml/v21.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index fb88d756f..b602eac9b 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -1248,6 +1248,7 @@ def _mds_start(reader, elem):
 
 @end("mes:MetadataSet", only=False)
 def _mds_end(reader, elem):
+    # Retrieve the current MetadataSet
     mds = reader.pop_single("MetadataSet")
 
     # Collect the contained MetadataReports
@@ -1260,24 +1261,20 @@ def _mds_end(reader, elem):
 @end(":Report md:Report")
 def _md_report(reader: Reader, elem):
     cls = reader.class_for_tag(elem.tag)
-
-    obj = cls(
+    return cls(
         attaches_to=reader.pop_single(model.TargetObjectKey),
-        metadata=reader.pop_all(model.ReportedAttribute, subclass=True),
+        metadata=reader.pop_single("AttributeSet"),
     )
-    return obj
 
 
 @end(":Target md:Target")
 def _tov(reader: Reader, elem):
     cls = reader.class_for_tag(elem.tag)
-
-    obj = cls(
+    return cls(
         key_values={
             v.value_for: v for v in reader.pop_all(v21.TargetObjectValue, subclass=True)
         }
     )
-    return obj
 
 
 @end(":ReferenceValue md:ReferenceValue")

From 70e63191f777b921226e2e025fa84862c997b13d Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:43:11 +0200
Subject: [PATCH 14/45] Improve .reader.xml.v21 handling of MetadataSet

- Stash/unstash on AttributeSet, not ReportedAttribute.
- Generate concrete subclasses of ReportedAttribute.
- Resolve references in TargetIdentifiableObject.
---
 sdmx/reader/xml/v21.py | 76 ++++++++++++++++++++++++++----------------
 1 file changed, 48 insertions(+), 28 deletions(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index b602eac9b..09d986e30 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -11,7 +11,7 @@
 from copy import copy
 from itertools import chain
 from sys import maxsize
-from typing import Any, Dict, Type, cast
+from typing import Any, Dict, MutableMapping, Type, cast
 
 from dateutil.parser import isoparse
 from lxml import etree
@@ -98,7 +98,6 @@ class Reader(XMLEventReader):
     "gen:ObsDimension gen:ObsValue gen:Value "
     # Tags that are bare containers for other XML elements
     """
-    :AttributeSet md:AttributeSet
     str:Categorisations str:CategorySchemes str:Codelists str:Concepts
     str:ConstraintAttachment str:Constraints str:CustomTypes str:Dataflows
     str:DataStructureComponents str:DataStructures str:FromVtlSuperSpace
@@ -1296,11 +1295,11 @@ def _rv(reader: Reader, elem):
     if cls is v21.TargetReportPeriod:
         args["report_period"] = reader.pop_single("ReportPeriod")
     elif cls is model.TargetIdentifiableObject:
-        args["obj"] = reader.pop_single("ObjectReference")
+        or_ = reader.pop_resolved_ref("ObjectReference")
+        reader.ignore.add(id(or_.parent))
+        args["obj"] = or_
 
-    obj = cls(**args)
-
-    return obj
+    return cls(**args)
 
 
 def add_mds_events(reader: Reader, mds: model.MetadataStructureDefinition):
@@ -1308,8 +1307,7 @@ def add_mds_events(reader: Reader, mds: model.MetadataStructureDefinition):
 
     # TODO these persist after reading a particular message; avoid this
     def _add_events_for_ma(ma: model.MetadataAttribute):
-        reader.start(f":{ma.id}", only=False)(_ra_start)
-        reader.end(f":{ma.id}", only=False)(_ra_end)
+        reader.end(f":{ma.id}")(_ra)
         for child in ma.child:
             _add_events_for_ma(child)
 
@@ -1318,33 +1316,55 @@ def _add_events_for_ma(ma: model.MetadataAttribute):
             _add_events_for_ma(ma)
 
 
-@start("md:ReportedAttribute", only=False)
-def _ra_start(reader: Reader, elem):
+@start(":AttributeSet md:AttributeSet", only=False)
+def _as_start(reader: Reader, elem):
     # Avoid collecting previous/sibling ReportedAttribute as children of this one
-    reader.stash(model.ReportedAttribute)
+    reader.stash("ReportedAttribute")
 
 
-@end("md:ReportedAttribute", only=False)
-def _ra_end(reader: Reader, elem):
-    cls = reader.class_for_tag(elem.tag)
-    if cls is None:
-        cls = reader.class_for_tag("md:ReportedAttribute")
-        value_for = elem.tag
-    else:
-        value_for = elem.attrib["id"]
+@end(":AttributeSet md:AttributeSet", only=False)
+def _as_end(reader: Reader, elem):
+    # Collect ReportedAttributes from the current AttributeSet in a list
+    reader.push("AttributeSet", reader.pop_all("ReportedAttribute"))
+    # Unstash others from the same level
+    reader.unstash()
 
-    # Pop all child elements
-    args = dict(child=reader.pop_all(cls, subclass=True), value_for=value_for)
 
-    xhtml = reader.pop_single("StructuredText")
-    if xhtml:
-        cls = v21.XHTMLAttributeValue
-        args["value"] = xhtml
+@end("md:ReportedAttribute")
+def _ra(reader: Reader, elem):
+    args: MutableMapping[str, Any] = dict()
 
-    obj = cls(**args)
+    # Unstash and retrieve child ReportedAttribute
+    child = reader.pop_single("AttributeSet")
+    if child:
+        args.update(child=child)
 
-    reader.unstash()
-    return obj
+    # TODO Match value_for to specific common.MetadataAttribute in the ReportStructure
+    # # Retrieve the current MetadataSet, MDSD, and ReportStructure
+    # mds = cast(model.MetadataSet, reader.get_single("MetadataSet"))
+
+    try:
+        # Set `value_for` using the "id" attribute
+        args["value_for"] = elem.attrib["id"]
+    except KeyError:
+        args["value_for"] = elem.tag
+
+    # Identify a concrete subclass of model.ReportedAttribute
+    xhtml_value_root_elem = reader.pop_single("StructuredText")
+    if xhtml_value_root_elem is not None:
+        cls: type = v21.XHTMLAttributeValue
+        args["value"] = xhtml_value_root_elem
+    else:
+        # TODO Distinguish model.EnumeratedAttributeValue
+        cls = model.OtherNonEnumeratedAttributeValue
+        try:
+            args["value"] = elem.attrib["value"]
+        except KeyError:
+            if not child:
+                raise
+
+    # Push onto a common ReportedAttribute stack; not a subclass-specific stack
+    reader.push("ReportedAttribute", cls(**args))
 
 
 # §8: Hierarchical Code List

From baa65bb4e5a579f471c96a1a46554085f4adf03e Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:43:57 +0200
Subject: [PATCH 15/45] Store XHTMLAttributeValue contents as etree nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…instead of str/bytes.
---
 sdmx/reader/xml/v21.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index 09d986e30..cf915c140 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -349,7 +349,7 @@ def _text(reader, elem):
 @start("com:StructuredText")
 def _st(reader, elem):
     """Contained XHTML."""
-    reader.push(elem, etree.tostring(elem[0], pretty_print=True))
+    reader.push(elem, elem[0])
 
 
 @end("mes:Extracted mes:Prepared mes:ReportingBegin mes:ReportingEnd")

From 08e3418c52546168ff77b1e671f69233832ebda6 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:44:22 +0200
Subject: [PATCH 16/45] Capitalize project URL titles for PyPI

---
 pyproject.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index a40f14b1a..43d29ca84 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,9 +50,9 @@ tests = [
 ]
 
 [project.urls]
-homepage = "https://github.com/khaeru/sdmx"
-repository = "https://github.com/khaeru/sdmx"
-documentation = "https://sdmx1.readthedocs.io/en/latest"
+Homepage = "https://github.com/khaeru/sdmx"
+Repository = "https://github.com/khaeru/sdmx"
+Documentation = "https://sdmx1.readthedocs.io/en/latest"
 
 [tool.coverage.run]
 omit = [

From 7dc758ebdb7889ef5ac872d0b02b76afd78beb4e Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Sun, 25 Aug 2024 18:46:22 +0200
Subject: [PATCH 17/45] Use https:// base URL for WB_WDI REST data source

---
 sdmx/sources.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdmx/sources.json b/sdmx/sources.json
index 18398fa9a..e20304ec7 100644
--- a/sdmx/sources.json
+++ b/sdmx/sources.json
@@ -431,7 +431,7 @@
   {
     "id": "WB_WDI",
     "name": "World Bank World Development Indicators",
-    "url": "http://api.worldbank.org/v2/sdmx/rest",
+    "url": "https://api.worldbank.org/v2/sdmx/rest",
     "supports": {
       "actualconstraint": false,
       "agencyscheme": false,

From 2b2c4d2fe2a11fe31b12e9ceb4a3324056ff775d Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Wed, 28 Aug 2024 13:24:26 +0200
Subject: [PATCH 18/45] Test round-trip of v21.MetadataStructure

---
 sdmx/tests/writer/test_writer_xml.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sdmx/tests/writer/test_writer_xml.py b/sdmx/tests/writer/test_writer_xml.py
index 8cb24d41a..fb50a56f0 100644
--- a/sdmx/tests/writer/test_writer_xml.py
+++ b/sdmx/tests/writer/test_writer_xml.py
@@ -261,6 +261,7 @@ def test_data_roundtrip(pytestconfig, specimen, data_id, structure_id, tmp_path)
         ("ECB/orgscheme.xml", True),
         ("ECB_EXR/1/structure-full.xml", False),
         ("ESTAT/apro_mk_cola-structure.xml", True),
+        ("ESTAT/esms-structure.xml", False),
         pytest.param(
             "ISTAT/47_850-structure.xml", True, marks=[pytest.mark.skip(reason="Slow")]
         ),

From 2101deb29496ca191f1c9eb910a8289abaf0ae92 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Wed, 28 Aug 2024 13:26:25 +0200
Subject: [PATCH 19/45] Write MetadataStructure to SDMX-ML StructureMessage

---
 sdmx/writer/xml.py | 38 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index d89197663..2a1fb76ed 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -14,7 +14,7 @@
 import sdmx.urn
 from sdmx import message
 from sdmx.format.xml.v21 import NS, qname, tag_for_class
-from sdmx.model import common
+from sdmx.model import common, v21
 from sdmx.model import v21 as model
 from sdmx.writer.base import BaseWriter
 
@@ -191,14 +191,17 @@ def _sm(obj: message.StructureMessage):
         ("concept_scheme", "Concepts"),
         ("structure", "DataStructures"),
         ("constraint", "Constraints"),
+        ("metadatastructure", "MetadataStructures"),
         ("provisionagreement", "ProvisionAgreements"),
     ]:
         coll = getattr(obj, attr)
         if not len(coll):
             continue
         container = Element(f"str:{tag}")
+
         for s in filter(lambda s: not s.is_external_reference, coll.values()):
             container.append(writer.recurse(s))
+
         if len(container):
             structures.append(container)
 
@@ -742,6 +745,39 @@ def _ds(obj: model.DataSet):
     return elem
 
 
+# SDMX 2.1 §7.3: Metadata Structure Definition
+
+
+@writer
+def _mdsd(obj: v21.MetadataStructureDefinition):
+    msc = Element(
+        "str:MetadataStructureComponents",
+        *[writer.recurse(mdt, obj) for mdt in obj.target.values()],
+        *[writer.recurse(rs, obj) for rs in obj.report_structure.values()],
+    )
+
+    return maintainable(obj, msc)
+
+
+@writer
+def _mda(obj: v21.MetadataAttribute, *args):
+    # Use the generic _component function to handle several common features
+    elem = _component(obj, *args)
+
+    # MetadataAttribute class properties
+    if obj.is_presentational is not None:
+        elem.attrib["isPresentational"] = str(obj.is_presentational).lower()
+    if obj.min_occurs is not None:
+        elem.attrib["minOccurs"] = str(obj.min_occurs)
+    if obj.max_occurs is not None:
+        elem.attrib["maxOccurs"] = str(obj.max_occurs)
+
+    # Recurse children
+    elem.extend([writer.recurse(mda, *args) for mda in obj.child])
+
+    return elem
+
+
 # SDMX 2.1 §7.4: Metadata Set
 
 

From 5bea0b25c694ad9c5a60a5201f41f489496581b0 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Wed, 28 Aug 2024 13:28:00 +0200
Subject: [PATCH 20/45] Remove style="Ref" XML attribute from Item

This was an erroneous inclusion, not part of the standard.
---
 sdmx/writer/xml.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 2a1fb76ed..668135bdf 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -352,11 +352,9 @@ def maintainable(obj, *args, **kwargs) -> etree._Element:
 def _item(obj: model.Item, **kwargs):
     elem = nameable(obj, **kwargs)
 
+    # Reference to parent Item
     if isinstance(obj.parent, obj.__class__):
-        # Reference to parent Item
-        e_parent = Element("str:Parent")
-        e_parent.append(Element(":Ref", id=obj.parent.id, style="Ref"))
-        elem.append(e_parent)
+        elem.append(Element("str:Parent", Element(":Ref", id=obj.parent.id)))
 
     if isinstance(obj, common.Organisation):
         elem.extend(writer.recurse(c) for c in obj.contact)

From 70bb5c818479cc73a9400af04642dc0facfc4b7d Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Wed, 28 Aug 2024 13:30:07 +0200
Subject: [PATCH 21/45] Adjust logging for compare() methods

- Log non-identical members of DictLike.
- Log missing components in other ComponentList.
---
 sdmx/dictlike.py     |  3 ++-
 sdmx/model/common.py | 11 ++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/sdmx/dictlike.py b/sdmx/dictlike.py
index 3c45b05f9..517008bab 100644
--- a/sdmx/dictlike.py
+++ b/sdmx/dictlike.py
@@ -98,11 +98,12 @@ def compare(self, other, strict=True):
             Passed to :func:`compare` for the values.
         """
         if set(self.keys()) != set(other.keys()):
-            log.info(f"Not identical: {sorted(self.keys())} / {sorted(other.keys())}")
+            log.debug(f"Not identical: {sorted(self.keys())} / {sorted(other.keys())}")
             return False
 
         for key, value in self.items():
             if not value.compare(other[key], strict):
+                log.debug(f"Not identical: {value} / {other[key]}")
                 return False
 
         return True
diff --git a/sdmx/model/common.py b/sdmx/model/common.py
index 3a0566440..fcd2a92fb 100644
--- a/sdmx/model/common.py
+++ b/sdmx/model/common.py
@@ -997,9 +997,14 @@ def compare(self, other, strict=True):
         strict : bool, optional
             Passed to :func:`.compare` and :meth:`.IdentifiableArtefact.compare`.
         """
-        return super().compare(other, strict) and all(
-            c.compare(other.get(c.id), strict) for c in self.components
-        )
+        result = True
+        for c in self.components:
+            try:
+                result &= c.compare(other.get(c.id), strict)
+            except KeyError:
+                log.debug(f"{other} has no component with ID {c.id!r}")
+                result = False
+        return result and super().compare(other, strict)
 
 
 # §4.5: Category Scheme

From 37e7f50ccb5821c40861972963b3da85e6c5c7ac Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Wed, 28 Aug 2024 13:32:26 +0200
Subject: [PATCH 22/45] =?UTF-8?q?Handle=20forward=20child=20Item=E2=86=92p?=
 =?UTF-8?q?arent=20refs=20in=20SDMX-ML?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 sdmx/reader/xml/v21.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index cf915c140..d7ca35cff 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -465,10 +465,16 @@ def _item_end(reader: Reader, elem):
             # Found 1 child XML element with same tag → claim 1 child object
             item.append_child(reader.pop_single(cls))
 
-    # (2) through <str:Parent>
-    parent = reader.pop_resolved_ref("Parent")
-    if parent:
-        parent.append_child(item)
+    # (2) through <str:Parent>. These may be backward or forward references. Backward
+    # references can be resolved through pop_resolved_ref(), but forward references
+    # cannot.
+    if parent := reader.pop_resolved_ref("Parent"):
+        if getattr(parent.get_scheme(), "is_external_reference", False):
+            # Forward reference
+            reader.push("item parent", (item.id, parent.id))
+        else:
+            # Backward reference
+            parent.append_child(item)
 
     # Agency only
     try:
@@ -516,6 +522,14 @@ def _itemscheme(reader: Reader, elem):
             #      this try/except can be removed.
             pass
 
+    # Add deferred forward references
+    for child_id, parent_id in reader.pop_all("item parent"):
+        try:
+            is_[parent_id].append_child(is_[child_id])
+        except KeyError:
+            if not is_.is_partial:
+                raise
+
     return is_
 
 

From 4e57f2ff4451ce0c2af4bd134fd428eb5794564b Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Wed, 28 Aug 2024 13:33:21 +0200
Subject: [PATCH 23/45] Read MetadataAttribute XML attributes

- Streamline .reader.v21._component_end()
---
 sdmx/reader/xml/v21.py | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index d7ca35cff..ee6cad6b7 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -11,7 +11,7 @@
 from copy import copy
 from itertools import chain
 from sys import maxsize
-from typing import Any, Dict, MutableMapping, Type, cast
+from typing import Any, Dict, MutableMapping, Optional, Type, cast
 
 from dateutil.parser import isoparse
 from lxml import etree
@@ -598,10 +598,14 @@ def _component_start(reader: Reader, elem):
     reader.stash(reader.class_for_tag(elem.tag))
 
 
+def _maybe_unbounded(value: str) -> Optional[int]:
+    return None if value == "unbounded" else int(value)
+
+
 @end(COMPONENT, only=False)
 @possible_reference(unstash=True)
 def _component_end(reader: Reader, elem):
-    # Object class: {,Measure,Time}Dimension or DataAttribute
+    # Object class: {,Measure,Time}Dimension; DataAttribute; MetadataAttribute
     cls = reader.class_for_tag(elem.tag)
 
     args = dict(
@@ -613,24 +617,30 @@ def _component_end(reader: Reader, elem):
         args["order"] = int(elem.attrib["position"])
     except KeyError:
         pass
+
     # DataAttributeOnly
-    us = elem.attrib.get("assignmentStatus")
-    if us:
+    if us := elem.attrib.get("assignmentStatus"):
         args["usage_status"] = model.UsageStatus[us.lower()]
 
-    cr = reader.pop_resolved_ref("ConceptRole")
-    if cr:
+    if cr := reader.pop_resolved_ref("ConceptRole"):
         args["concept_role"] = cr
 
     # DataAttribute only
-    ar = reader.pop_all(model.AttributeRelationship, subclass=True)
-    if len(ar):
+    if ar := reader.pop_all(model.AttributeRelationship, subclass=True):
         assert len(ar) == 1, ar
         args["related_to"] = ar[0]
 
-    # MetadataAttribute.child only
-    if children := reader.pop_all(cls):
-        args["child"] = children
+    if cls is v21.MetadataAttribute:
+        setdefault_attrib(args, elem, "isPresentational", "maxOccurs", "minOccurs")
+        if "is_presentational" in args:
+            args["is_presentational"] = bool(args["is_presentational"])
+
+        if "max_occurs" in args:
+            args["max_occurs"] = _maybe_unbounded(args["max_occurs"])
+        if "min_occurs" in args:
+            args["min_occurs"] = _maybe_unbounded(args["min_occurs"])
+        if children := reader.pop_all(cls):
+            args["child"] = children
 
     reader.unstash()
 
@@ -639,10 +649,7 @@ def _component_end(reader: Reader, elem):
     # assumed to be the same as the identifier of the concept referenced from the
     # concept identity.”
     if args["id"] is common.MissingID:
-        try:
-            args["id"] = args["concept_identity"].id
-        except AttributeError:
-            pass
+        args["id"] = getattr(args["concept_identity"], "id", None) or args["id"]
 
     return reader.identifiable(cls, elem, **args)
 

From e94f08090d568e67b128fd9b790736174871edff Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Wed, 28 Aug 2024 13:35:17 +0200
Subject: [PATCH 24/45] Improve XMLEventReader.maintainable()

Handle the case where the existing object is concrete, not an external
reference, but has no URN set (e.g. because not given in the source
message) by computing a URN.
---
 sdmx/reader/xml/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sdmx/reader/xml/common.py b/sdmx/reader/xml/common.py
index d26de88c6..929e71d9e 100644
--- a/sdmx/reader/xml/common.py
+++ b/sdmx/reader/xml/common.py
@@ -579,10 +579,13 @@ def maintainable(self, cls, elem, **kwargs):
         existing = self.get_single(cls, obj.id, version=obj.version)
 
         if existing and (
-            existing.compare(obj, strict=True) or existing.urn == sdmx.urn.make(obj)
+            existing.compare(obj, strict=True)
+            or (existing.urn or sdmx.urn.make(existing)) == sdmx.urn.make(obj)
         ):
             if elem is not None:
                 # Previously an external reference, now concrete
+                # FIXME Handle the case where a MaintainableArtefact is included in a
+                #       StructureMessage, with is_external_reference explicitly True
                 existing.is_external_reference = False
 
                 # Update `existing` from `obj` to preserve references
@@ -616,6 +619,7 @@ def matching_class(cls):
 
 
 def setdefault_attrib(target, elem, *names):
+    """Update `target` from :py:`elem.attrib` for the given `names`."""
     try:
         for name in names:
             try:

From f5e4a7ee207788e8a51801bb6c2676144ab25a0a Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Mon, 2 Sep 2024 20:27:32 +0200
Subject: [PATCH 25/45] Raise NIE for MetadataSet in .writer.pandas

---
 sdmx/writer/pandas.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/sdmx/writer/pandas.py b/sdmx/writer/pandas.py
index 0bd8f0886..80c57feb5 100644
--- a/sdmx/writer/pandas.py
+++ b/sdmx/writer/pandas.py
@@ -545,7 +545,13 @@ def _mv(obj: model.MemberValue):
 
 
 @writer
-def _na(obj: model.NameableArtefact):
+def _mds(obj: model.MetadataSet, **kwargs):
+    raise NotImplementedError(f"write {type(obj).__name__} to pandas")
+
+
+@writer
+def _na(obj: model.NameableArtefact, **kwargs):
+    """Fallback for NameableArtefact: only its name."""
     return str(obj.name)
 
 

From 111f602152fdabe3fe93577317a5ce4c97b4a801 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Mon, 2 Sep 2024 20:30:10 +0200
Subject: [PATCH 26/45] Improve .writer.base.BaseWriter

- Distinguish NoWriterImplementation from NotImplementedError.
- When dispatching to a writer function for the parent class of an
  object, register that function for the child class directly.
---
 sdmx/writer/base.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/sdmx/writer/base.py b/sdmx/writer/base.py
index 3a318947c..efa9b8dc6 100644
--- a/sdmx/writer/base.py
+++ b/sdmx/writer/base.py
@@ -1,6 +1,10 @@
 from functools import singledispatch
 
 
+class NoWriterImplementation(NotImplementedError):
+    pass
+
+
 class BaseWriter:
     """Base class for recursive writers.
 
@@ -32,9 +36,7 @@ def __init__(self, format_name):
         # Create the single-dispatch function
         @singledispatch
         def func(obj, *args, **kwargs):
-            raise NotImplementedError(
-                f"write {obj.__class__.__name__} to " f"{format_name}"
-            )
+            raise NoWriterImplementation(f"write {type(obj).__name__} to {format_name}")
 
         self._dispatcher = func
 
@@ -44,20 +46,21 @@ def recurse(self, obj, *args, **kwargs):
         If there is no :meth:`register` 'ed function to write the class of `obj`, then
         the parent class of `obj` is used to find a method.
         """
-        # TODO use a cache to speed up, so the MRO does not need to be traversed for
-        #      every object instance
-
         dispatcher = getattr(self, "_dispatcher")
+
         try:
             # Let the single dispatch function choose the overload
             return dispatcher(obj, *args, **kwargs)
-        except NotImplementedError as exc:
+        except NoWriterImplementation as exc:
             try:
-                # Use the object's parent class to get a different overload
-                func = dispatcher.registry[obj.__class__.mro()[1]]
+                # Use the object's parent class to get a different implementation
+                cls = type(obj).mro()[1]
+                func = dispatcher.registry[cls]
             except KeyError:
-                # Overload for the parent class did not exist
-                raise exc
+                raise exc  # No implementation for the parent class
+            else:
+                # Success; register the function so it is found directly next time
+                dispatcher.register(type(obj), func)
 
             return func(obj, *args, **kwargs)
 

From 3b8ae1ea58ca761c687ca5e377de7b67abcca47d Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Mon, 2 Sep 2024 20:30:58 +0200
Subject: [PATCH 27/45] Silence warnings about provider= kwarg in tests

---
 sdmx/tests/test_docs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdmx/tests/test_docs.py b/sdmx/tests/test_docs.py
index 6f7ef2eef..0bda71421 100644
--- a/sdmx/tests/test_docs.py
+++ b/sdmx/tests/test_docs.py
@@ -128,7 +128,7 @@ def test_doc_usage_structure():
         )
     )
 
-    msg1 = ecb.categoryscheme(provider="all")
+    msg1 = ecb.categoryscheme(agency_id="all")
 
     assert msg1.response.url == (
         "https://data-api.ecb.europa.eu/service/categoryscheme/all/all/latest"

From 2266e8729c6215b1a2582210d23ff84c79a0d50f Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Mon, 2 Sep 2024 20:32:13 +0200
Subject: [PATCH 28/45] Improve SpecimenCollection.as_params(..., marks=...)

Match on a string path fragment (same as test case ID), rather than a
complete path object.
---
 sdmx/testing/__init__.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/sdmx/testing/__init__.py b/sdmx/testing/__init__.py
index 773b9ad78..c32e7769f 100644
--- a/sdmx/testing/__init__.py
+++ b/sdmx/testing/__init__.py
@@ -2,7 +2,7 @@
 import os
 from collections import ChainMap
 from contextlib import contextmanager
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import List, Tuple, Union
 
 import numpy as np
@@ -349,13 +349,17 @@ def as_params(self, format=None, kind=None, marks=dict()):
         `format` and `kind` arguments (if any). Marks are attached to each param from
         `marks`, wherein the keys are partial paths.
         """
+        # Transform `marks` into a platform-independent mapping from path parts
+        _marks = {PurePosixPath(k).parts: v for k, v in marks.items()}
+
         for path, f, k in self.specimens:
             if (format and format != f) or (kind and kind != k):
                 continue
+            p_rel = path.relative_to(self.base_path)
             yield pytest.param(
                 path,
-                id=str(path.relative_to(self.base_path)),
-                marks=marks.get(path, tuple()),
+                id=str(p_rel),  # String ID for this specimen
+                marks=_marks.get(p_rel.parts, tuple()),  # Look up marks via path parts
             )
 
     def expected_data(self, path):

From 223f7885512f6c1f25817e34d814c5d31f5e9d4a Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Mon, 2 Sep 2024 20:32:41 +0200
Subject: [PATCH 29/45] Xfail writing MetadataSet to pandas, CSV

---
 sdmx/tests/writer/test_csv.py    | 6 +++++-
 sdmx/tests/writer/test_pandas.py | 8 ++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/sdmx/tests/writer/test_csv.py b/sdmx/tests/writer/test_csv.py
index cc8d0216c..456022402 100644
--- a/sdmx/tests/writer/test_csv.py
+++ b/sdmx/tests/writer/test_csv.py
@@ -4,6 +4,10 @@
 import sdmx
 from sdmx.model import v21 as m
 
+MARKS = {
+    "ESTAT/esms.xml": pytest.mark.xfail(raises=NotImplementedError),
+}
+
 
 def _add_test_dsd(ds: m.DataSet) -> None:
     if ds.described_by is None:
@@ -17,7 +21,7 @@ def _add_test_dsd(ds: m.DataSet) -> None:
             )
 
 
-@pytest.mark.parametrize_specimens("path", kind="data")
+@pytest.mark.parametrize_specimens("path", kind="data", marks=MARKS)
 def test_write_data(tmp_path, specimen, path):
     msg = sdmx.read_sdmx(path)
 
diff --git a/sdmx/tests/writer/test_pandas.py b/sdmx/tests/writer/test_pandas.py
index 96d764b53..452b028b3 100644
--- a/sdmx/tests/writer/test_pandas.py
+++ b/sdmx/tests/writer/test_pandas.py
@@ -8,6 +8,10 @@
 from sdmx.model.v21 import TimeDimension
 from sdmx.testing import assert_pd_equal
 
+MARKS = {
+    "ESTAT/esms.xml": pytest.mark.xfail(raises=NotImplementedError),
+}
+
 
 def test_write_data_arguments(specimen):
     # The identity here is not important; any non-empty DataMessage will work
@@ -23,7 +27,7 @@ def test_write_data_arguments(specimen):
         sdmx.to_pandas(msg, attributes="foobarbaz")
 
 
-@pytest.mark.parametrize_specimens("path", kind="data")
+@pytest.mark.parametrize_specimens("path", kind="data", marks=MARKS)
 def test_write_data(specimen, path):
     msg = sdmx.read_sdmx(path)
 
@@ -38,7 +42,7 @@ def test_write_data(specimen, path):
     assert isinstance(result, (pd.Series, pd.DataFrame, list)), type(result)
 
 
-@pytest.mark.parametrize_specimens("path", kind="data")
+@pytest.mark.parametrize_specimens("path", kind="data", marks=MARKS)
 def test_write_data_attributes(path):
     msg = sdmx.read_sdmx(path)
 

From 879f0817970e915c73f12135bbb446d39b1877ea Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Mon, 2 Sep 2024 20:34:38 +0200
Subject: [PATCH 30/45] Relax strict comparison of IdentifiableArtefact

Allow non-strict comparison if self.urn is None.
---
 sdmx/model/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sdmx/model/common.py b/sdmx/model/common.py
index fcd2a92fb..1a509a070 100644
--- a/sdmx/model/common.py
+++ b/sdmx/model/common.py
@@ -287,7 +287,8 @@ def compare(self, other, strict=True):
         return (
             compare("id", self, other, strict)
             and compare("uri", self, other, strict)
-            and compare("urn", self, other, strict)
+            # Allow non-strict comparison if self.urn is None
+            and compare("urn", self, other, strict and self.urn is not None)
         )
 
     def __hash__(self):

From e1474dd4e3c3a8094fe9111bda6208f3b9aa024c Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:35:25 +0200
Subject: [PATCH 31/45] Add IdentifiableObjectTarget.{object_type,compare()}

---
 sdmx/model/v21.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/sdmx/model/v21.py b/sdmx/model/v21.py
index 5b57a1cf9..0bbbc21d1 100644
--- a/sdmx/model/v21.py
+++ b/sdmx/model/v21.py
@@ -19,6 +19,7 @@
 )
 
 from sdmx.dictlike import DictLikeDescriptor
+from sdmx.util import compare
 
 from . import common
 from .common import (
@@ -344,9 +345,29 @@ class DimensionDescriptorValuesTarget(TargetObject):
     """SDMX 2.1 DimensionDescriptorValuesTarget."""
 
 
+@dataclass
 class IdentifiableObjectTarget(TargetObject):
     """SDMX 2.1 IdentifiableObjectTarget."""
 
+    #: Type of :class:`.IdentifiableArtefact` that is targeted.
+    object_type: Optional[Type[IdentifiableArtefact]] = None
+
+    def compare(self, other, strict=True):
+        """Return :obj:`True` if `self` is the same as `other`.
+
+        Two IdentifiableObjectTargets are the same if
+        :meth:`.IdentifiableArtefact.compare` is :obj:`True` and they have the same
+        :attr:`object_type`.
+
+        Parameters
+        ----------
+        strict : bool, optional
+            Passed to :func:`.compare`.
+        """
+        return super().compare(other, strict) and compare(
+            "object_type", self, other, strict
+        )
+
 
 class ReportPeriodTarget(TargetObject):
     """SDMX 2.1 ReportPeriodTarget."""

From dd979d31f533b37cc386c62c61c5d41515618ddc Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:40:36 +0200
Subject: [PATCH 32/45] Make round-trip test of esms-structure.xml strict

---
 sdmx/tests/writer/test_writer_xml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdmx/tests/writer/test_writer_xml.py b/sdmx/tests/writer/test_writer_xml.py
index fb50a56f0..93ad2af43 100644
--- a/sdmx/tests/writer/test_writer_xml.py
+++ b/sdmx/tests/writer/test_writer_xml.py
@@ -261,7 +261,7 @@ def test_data_roundtrip(pytestconfig, specimen, data_id, structure_id, tmp_path)
         ("ECB/orgscheme.xml", True),
         ("ECB_EXR/1/structure-full.xml", False),
         ("ESTAT/apro_mk_cola-structure.xml", True),
-        ("ESTAT/esms-structure.xml", False),
+        ("ESTAT/esms-structure.xml", True),
         pytest.param(
             "ISTAT/47_850-structure.xml", True, marks=[pytest.mark.skip(reason="Slow")]
         ),

From 8d4194c63b3ffff2adeb1ac9c9ea6d7abc6c272e Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:41:02 +0200
Subject: [PATCH 33/45] Add .util.ucfirst()

---
 sdmx/util/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sdmx/util/__init__.py b/sdmx/util/__init__.py
index 2e055e52e..889715edb 100644
--- a/sdmx/util/__init__.py
+++ b/sdmx/util/__init__.py
@@ -80,6 +80,11 @@ def parse_content_type(value: str) -> Tuple[str, Dict[str, Any]]:
     return content_type, params
 
 
+def ucfirst(value: str) -> str:
+    """Return `value` with its first character transformed to upper-case."""
+    return value[0].upper() + value[1:]
+
+
 _FIELDS_CACHE: Dict[str, List[Field]] = dict()
 
 

From 27e35cea9e14a4a9463b9d45cc5432e3498b59c2 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:41:47 +0200
Subject: [PATCH 34/45] Write FacetValueType to XML in CamelCase

---
 sdmx/writer/xml.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 668135bdf..b5edafb0e 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -16,6 +16,7 @@
 from sdmx.format.xml.v21 import NS, qname, tag_for_class
 from sdmx.model import common, v21
 from sdmx.model import v21 as model
+from sdmx.util import ucfirst
 from sdmx.writer.base import BaseWriter
 
 _element_maker = ElementMaker(nsmap={k: v for k, v in NS.items() if v is not None})
@@ -377,8 +378,12 @@ def _is(obj: model.ItemScheme):
 
 @writer
 def _facet(obj: model.Facet):
-    # TODO textType should be CamelCase
-    return Element("str:TextFormat", textType=getattr(obj.value_type, "name", None))
+    attrib: MutableMapping[str, str] = dict()
+    try:
+        attrib.update(textType=ucfirst(obj.value_type.name))
+    except AttributeError:
+        pass
+    return Element("str:TextFormat", **attrib)
 
 
 @writer

From f8387b93f7187bb80c7596160324888d0c2ebefb Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:42:19 +0200
Subject: [PATCH 35/45] Write ItemScheme.is_partial to XML

---
 sdmx/writer/xml.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index b5edafb0e..3d52df1c1 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -365,7 +365,10 @@ def _item(obj: model.Item, **kwargs):
 
 @writer
 def _is(obj: model.ItemScheme):
-    elem = maintainable(obj)
+    kw = dict()
+    if obj.is_partial is not None:
+        kw["isPartial"] = str(obj.is_partial).lower()
+    elem = maintainable(obj, **kw)
 
     # Pass _with_urn to identifiable(): don't generate URNs for Items in `obj` which do
     # not already have them

From c9477b033999d3b0067889e6256c3925360dfcb7 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:44:50 +0200
Subject: [PATCH 36/45] Add specific XML writers for 3 classes

 - ReportStructure
 - IdentifiableObjectTarget
 - ReportPeriodTarget
---
 sdmx/writer/xml.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 3d52df1c1..f8398075f 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -765,6 +765,18 @@ def _mdsd(obj: v21.MetadataStructureDefinition):
     return maintainable(obj, msc)
 
 
+@writer
+def _rs(obj: v21.ReportStructure):
+    elem = _cl(obj, None)
+    elem.extend(
+        [
+            Element("str:MetadataTarget", Element(":Ref", id=mdt.id))
+            for mdt in obj.report_for
+        ]
+    )
+    return elem
+
+
 @writer
 def _mda(obj: v21.MetadataAttribute, *args):
     # Use the generic _component function to handle several common features
@@ -784,6 +796,25 @@ def _mda(obj: v21.MetadataAttribute, *args):
     return elem
 
 
+@writer
+def _iot(obj: v21.IdentifiableObjectTarget, *args):
+    # IdentifiableObjectTarget class properties
+    attrib: MutableMapping[str, str] = dict()
+    if obj.object_type:
+        attrib.update(objectType=str(obj.object_type.__name__))
+
+    # Use the generic _component function to handle several common features
+    return _component(obj, *args, attrib=attrib)
+
+
+@writer
+def _rpt(obj: v21.ReportPeriodTarget, *args):
+    elem = _component(obj, *args)
+    # Do not write "id" attribute
+    elem.attrib.pop("id")
+    return elem
+
+
 # SDMX 2.1 §7.4: Metadata Set
 
 

From 3667d7bd641fc9077ecda22157b0b06b33154fa2 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:46:01 +0200
Subject: [PATCH 37/45] =?UTF-8?q?Add=20.writer.xml.=5Fcomponent(=E2=80=A6,?=
 =?UTF-8?q?=20attrib=3D=E2=80=A6)=20kwarg?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 sdmx/writer/xml.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index f8398075f..398c647e5 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -6,7 +6,7 @@
 # - writer functions for sdmx.model classes, in the same order as model.py
 
 import logging
-from typing import Iterable, List, Literal, MutableMapping
+from typing import Iterable, List, Literal, MutableMapping, Optional
 
 from lxml import etree
 from lxml.builder import ElementMaker
@@ -432,9 +432,9 @@ def _contact(obj: model.Contact):
 
 
 @writer
-def _component(obj: model.Component, dsd):
+def _component(obj: model.Component, dsd, *, attrib: Optional[dict] = None):
     child = []
-    attrib = dict()
+    attrib = attrib or dict()
 
     try:
         child.append(

From e76189e44c285e2cb5cc803b98cade54be1a2d86 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:46:42 +0200
Subject: [PATCH 38/45] Simplify XML writers for metadata structures

---
 sdmx/writer/xml.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 398c647e5..55a824cb5 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -759,7 +759,7 @@ def _mdsd(obj: v21.MetadataStructureDefinition):
     msc = Element(
         "str:MetadataStructureComponents",
         *[writer.recurse(mdt, obj) for mdt in obj.target.values()],
-        *[writer.recurse(rs, obj) for rs in obj.report_structure.values()],
+        *[writer.recurse(rs) for rs in obj.report_structure.values()],
     )
 
     return maintainable(obj, msc)
@@ -779,16 +779,17 @@ def _rs(obj: v21.ReportStructure):
 
 @writer
 def _mda(obj: v21.MetadataAttribute, *args):
-    # Use the generic _component function to handle several common features
-    elem = _component(obj, *args)
-
     # MetadataAttribute class properties
+    attrib = dict()
     if obj.is_presentational is not None:
-        elem.attrib["isPresentational"] = str(obj.is_presentational).lower()
+        attrib["isPresentational"] = str(obj.is_presentational).lower()
     if obj.min_occurs is not None:
-        elem.attrib["minOccurs"] = str(obj.min_occurs)
+        attrib["minOccurs"] = str(obj.min_occurs)
     if obj.max_occurs is not None:
-        elem.attrib["maxOccurs"] = str(obj.max_occurs)
+        attrib["maxOccurs"] = str(obj.max_occurs)
+
+    # Use the generic _component function to handle several common features
+    elem = _component(obj, *args, attrib=attrib)
 
     # Recurse children
     elem.extend([writer.recurse(mda, *args) for mda in obj.child])

From 73cca938406980f81417bceec57c3caf9a5e3613 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:49:08 +0200
Subject: [PATCH 39/45] Refine handling of MaintainableArtefact from XML

Handle cases where objects are included explicitly within a
StructureMessage, but with isExternalReference="true", i.e. as
"stubs".
---
 sdmx/reader/xml/common.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/sdmx/reader/xml/common.py b/sdmx/reader/xml/common.py
index 929e71d9e..d7d0603af 100644
--- a/sdmx/reader/xml/common.py
+++ b/sdmx/reader/xml/common.py
@@ -545,7 +545,6 @@ def maintainable(self, cls, elem, **kwargs):
         the is_external_reference attribute set to :obj:`True`. Subsequent calls with
         the same object ID will return references to the same object.
         """
-        kwargs.setdefault("is_external_reference", elem is None)
         setdefault_attrib(
             kwargs,
             elem,
@@ -555,6 +554,12 @@ def maintainable(self, cls, elem, **kwargs):
             "validTo",
             "version",
         )
+
+        # Ensure is_external_reference and is_final are bool
+        try:
+            kwargs["is_external_reference"] = kwargs["is_external_reference"] == "true"
+        except (KeyError, SyntaxError):
+            kwargs.setdefault("is_external_reference", elem is None)
         kwargs["is_final"] = kwargs.get("is_final", None) == "true"
 
         # Create a candidate object
@@ -583,18 +588,9 @@ def maintainable(self, cls, elem, **kwargs):
             or (existing.urn or sdmx.urn.make(existing)) == sdmx.urn.make(obj)
         ):
             if elem is not None:
-                # Previously an external reference, now concrete
-                # FIXME Handle the case where a MaintainableArtefact is included in a
-                #       StructureMessage, with is_external_reference explicitly True
-                existing.is_external_reference = False
-
                 # Update `existing` from `obj` to preserve references
                 # If `existing` was a forward reference <Ref/>, its URN was not stored.
                 for attr in list(kwargs.keys()) + ["urn"]:
-                    # log.info(
-                    #     f"Updating {attr} {getattr(existing, attr)} "
-                    #     f"{getattr(obj, attr)}"
-                    # )
                     setattr(existing, attr, getattr(obj, attr))
 
             # Discard the candidate
@@ -627,7 +623,7 @@ def setdefault_attrib(target, elem, *names):
             except KeyError:
                 pass
     except AttributeError:
-        pass
+        pass  # No elem.attrib; elem is None
 
 
 TO_SNAKE_RE = re.compile("([A-Z]+)")

From 6f82d255757ad7dfd7600485cbad740443ffd98f Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:50:49 +0200
Subject: [PATCH 40/45] Read IdentifiableObjectTarget from XML

---
 sdmx/reader/xml/v21.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index ee6cad6b7..8f4d94495 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -602,10 +602,10 @@ def _maybe_unbounded(value: str) -> Optional[int]:
     return None if value == "unbounded" else int(value)
 
 
+# TODO Reduce complexity from 12 → 11, by adding separate parsers for certain COMPONENTs
 @end(COMPONENT, only=False)
 @possible_reference(unstash=True)
-def _component_end(reader: Reader, elem):
-    # Object class: {,Measure,Time}Dimension; DataAttribute; MetadataAttribute
+def _component_end(reader: Reader, elem):  # noqa: C901
     cls = reader.class_for_tag(elem.tag)
 
     args = dict(
@@ -613,10 +613,8 @@ def _component_end(reader: Reader, elem):
         concept_identity=reader.pop_resolved_ref("ConceptIdentity"),
         local_representation=reader.pop_single(common.Representation),
     )
-    try:
-        args["order"] = int(elem.attrib["position"])
-    except KeyError:
-        pass
+    if position := elem.attrib.get("position"):
+        args["order"] = int(position)
 
     # DataAttributeOnly
     if us := elem.attrib.get("assignmentStatus"):
@@ -634,13 +632,13 @@ def _component_end(reader: Reader, elem):
         setdefault_attrib(args, elem, "isPresentational", "maxOccurs", "minOccurs")
         if "is_presentational" in args:
             args["is_presentational"] = bool(args["is_presentational"])
-
-        if "max_occurs" in args:
-            args["max_occurs"] = _maybe_unbounded(args["max_occurs"])
-        if "min_occurs" in args:
-            args["min_occurs"] = _maybe_unbounded(args["min_occurs"])
+        for name in "max_occurs", "min_occurs":
+            if name in args:
+                args[name] = _maybe_unbounded(args[name])
         if children := reader.pop_all(cls):
             args["child"] = children
+    elif cls is v21.IdentifiableObjectTarget:
+        args["object_type"] = model.get_class(elem.attrib["objectType"])
 
     reader.unstash()
 

From 4ba06eec5b239ea52d7c9f7d355bc3aae8b287e8 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 10:52:34 +0200
Subject: [PATCH 41/45] Expand TestUNICEF.test_data

---
 sdmx/tests/test_sources.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/sdmx/tests/test_sources.py b/sdmx/tests/test_sources.py
index 48aaa7a48..81eccdb9f 100644
--- a/sdmx/tests/test_sources.py
+++ b/sdmx/tests/test_sources.py
@@ -634,9 +634,18 @@ class TestUNICEF(DataSourceTest):
 
     @pytest.mark.network
     def test_data(self, client):
-        dsd = client.dataflow("GLOBAL_DATAFLOW").structure[0]
+        dm = client.dataflow("GLOBAL_DATAFLOW")
+        dsd = dm.structure[0]
+
         client.data("GLOBAL_DATAFLOW", key="ALB+DZA.MNCH_INSTDEL.", dsd=dsd)
 
+        cl = dm.codelist["CL_UNICEF_INDICATOR"]
+        c = cl["TRGT_2030_CME_MRM0"]
+
+        # Code is properly associated with its parent, despite forward reference
+        assert isinstance(c.parent, type(c))
+        assert "TRGT_CME" == c.parent.id
+
     @pytest.mark.network
     def test_cd2030(self, client):
         """Test that :ref:`Countdown to 2030 <CD2030>` data can be queried."""

From d31322f2503e41edc85c09971c322aed1b24393e Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 11:27:50 +0200
Subject: [PATCH 42/45] Document {v21,v30}.MetadataSet.structured_by

---
 sdmx/model/common.py | 7 +++++++
 sdmx/model/v21.py    | 3 +++
 sdmx/model/v30.py    | 8 ++++++++
 3 files changed, 18 insertions(+)

diff --git a/sdmx/model/common.py b/sdmx/model/common.py
index 1a509a070..16efb70cd 100644
--- a/sdmx/model/common.py
+++ b/sdmx/model/common.py
@@ -2157,6 +2157,13 @@ class BaseMetadataSet:
     publication_period: Optional[date] = None
     publication_year: Optional[date] = None
 
+    described_by: Optional[BaseMetadataflow] = None
+
+    #: Note that the class of this attribute differs from SDMX 2.1 to SDMX 3.0.
+    #: Compare :attr:`.v21.MetadataSet.structured_by` and
+    #: :attr:`.v30.MetadataSet.structured_by`.
+    structured_by: Optional[IdentifiableArtefact] = None
+
 
 # SDMX 2.1 §8: Hierarchical Code List
 # SDMX 3.0 §8: Hierarchy
diff --git a/sdmx/model/v21.py b/sdmx/model/v21.py
index 0bbbc21d1..8779c5734 100644
--- a/sdmx/model/v21.py
+++ b/sdmx/model/v21.py
@@ -513,6 +513,9 @@ class MetadataSet(NameableArtefact, common.BaseMetadataSet):
 
     described_by: Optional[MetadataflowDefinition] = None
     # described_by: Optional[ReportStructure] = None
+
+    #: .. seealso:: :attr:`.v30.MetadataSet.structured_by`, which has different
+    #:    semantics.
     structured_by: Optional[MetadataStructureDefinition] = None
 
     #: Analogous to :attr:`.v30.MetadataSet.provided_by`.
diff --git a/sdmx/model/v30.py b/sdmx/model/v30.py
index 871cf68b7..fa6d2f11c 100644
--- a/sdmx/model/v30.py
+++ b/sdmx/model/v30.py
@@ -486,10 +486,18 @@ class MetadataSet(MaintainableArtefact, common.BaseMetadataSet):
     valid_to: Optional[str] = None
     set_id: Optional[str] = None
 
+    #: .. note:: According to the standard, MetadataSet has **two** associations, both
+    #:    named :py:`.described_by`: one to a :class:`.Metadataflow`, and the other to a
+    #:    :class:`.MetadataProvisionAgreement`. :mod:`sdmx` implements the first,
+    #:    because it is consistent with SDMX 2.1.
     described_by: Optional[Metadataflow] = None
 
     # described_by: Optional[MetadataProvisionAgreement] = None
 
+    #: .. note:: According to the standard, this differs from
+    #:    :attr:`v21.MetadataSet.structured_by` in pointing to
+    #:    :attr:`.MetadataStructureDefinition.attributes`, directly, rather than to the
+    #:    MetadataStructureDefinition itself.
     structured_by: Optional[MetadataAttributeDescriptor] = None
 
     #: Analogous to :attr:`.v21.MetadataSet.published_by`.

From 7eea584558ea0cc63a4a9845acd85afefc7df746 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 11:28:18 +0200
Subject: [PATCH 43/45] Add MSD to .rest.CLASS_NAME and test

---
 sdmx/rest/common.py      | 1 +
 sdmx/tests/test_model.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/sdmx/rest/common.py b/sdmx/rest/common.py
index 304ea1605..d1190f804 100644
--- a/sdmx/rest/common.py
+++ b/sdmx/rest/common.py
@@ -15,6 +15,7 @@
 CLASS_NAME = {
     "dataflow": "DataflowDefinition",
     "datastructure": "DataStructureDefinition",
+    "metadatastructure": "MetadataStructureDefinition",
 }
 
 # Inverse of :data:`CLASS_NAME`.
diff --git a/sdmx/tests/test_model.py b/sdmx/tests/test_model.py
index fc1817035..629ca2089 100644
--- a/sdmx/tests/test_model.py
+++ b/sdmx/tests/test_model.py
@@ -216,6 +216,7 @@ def test_complete(module, extra):
         (dict(name=Resource.conceptscheme), model.ConceptScheme),
         (dict(name=Resource.contentconstraint), v21.ContentConstraint),
         (dict(name=Resource.dataflow), v21.DataflowDefinition),
+        (dict(name=Resource.metadatastructure), v21.MetadataStructureDefinition),
         (dict(name=Resource.organisationscheme), model.OrganisationScheme),
         (dict(name=Resource.provisionagreement), v21.ProvisionAgreement),
         pytest.param(

From ef8d9d937e358a7226cd1059c4223a2142c33d1f Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 12:25:20 +0200
Subject: [PATCH 44/45] Expand tests for #192 to maintain coverage

---
 sdmx/reader/xml/v21.py          |  4 ++--
 sdmx/tests/model/test_common.py | 18 ++++++++++++++++++
 sdmx/writer/xml.py              | 18 ++++++++++--------
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/sdmx/reader/xml/v21.py b/sdmx/reader/xml/v21.py
index 8f4d94495..f246ccdc3 100644
--- a/sdmx/reader/xml/v21.py
+++ b/sdmx/reader/xml/v21.py
@@ -527,7 +527,7 @@ def _itemscheme(reader: Reader, elem):
         try:
             is_[parent_id].append_child(is_[child_id])
         except KeyError:
-            if not is_.is_partial:
+            if not is_.is_partial:  # pragma: no cover
                 raise
 
     return is_
@@ -1379,7 +1379,7 @@ def _ra(reader: Reader, elem):
         try:
             args["value"] = elem.attrib["value"]
         except KeyError:
-            if not child:
+            if not child:  # pragma: no cover
                 raise
 
     # Push onto a common ReportedAttribute stack; not a subclass-specific stack
diff --git a/sdmx/tests/model/test_common.py b/sdmx/tests/model/test_common.py
index 03de117fb..e5fde3627 100644
--- a/sdmx/tests/model/test_common.py
+++ b/sdmx/tests/model/test_common.py
@@ -10,6 +10,8 @@
     Agency,
     AnnotableArtefact,
     Annotation,
+    Component,
+    ComponentList,
     Contact,
     IdentifiableArtefact,
     Item,
@@ -343,6 +345,22 @@ def test_repr(self) -> None:
         assert "<Representation: is0, []>" == repr(r)
 
 
+class TestComponentList:
+    def test_compare(self, caplog) -> None:
+        """Test comparison of two CL with mismatched components."""
+
+        components = [Component(id=s) for s in ("FOO", "BAR", "BAZ")]
+
+        cl1: ComponentList = ComponentList(id="CL", components=components)
+        cl2: ComponentList = ComponentList(id="CL", components=components[:-1])
+
+        # cl1 and cl2 compare as different
+        assert False is cl1.compare(cl2)
+
+        # Log message is emitted for mismatched components
+        assert "CL has no component with ID 'BAZ'" in caplog.messages
+
+
 class TestContact:
     def test_init(self):
         Contact(
diff --git a/sdmx/writer/xml.py b/sdmx/writer/xml.py
index 55a824cb5..1dac8cc1e 100644
--- a/sdmx/writer/xml.py
+++ b/sdmx/writer/xml.py
@@ -294,7 +294,7 @@ def _a(obj: model.Annotation):
 def annotable(obj, *args, **kwargs) -> etree._Element:
     # Determine tag
     tag = kwargs.pop("_tag", tag_for_class(obj.__class__))
-    if tag is None:
+    if tag is None:  # pragma: no cover
         raise NotImplementedError(f"Write {obj.__class__} to SDMX-ML")
 
     # Write Annotations
@@ -384,7 +384,7 @@ def _facet(obj: model.Facet):
     attrib: MutableMapping[str, str] = dict()
     try:
         attrib.update(textType=ucfirst(obj.value_type.name))
-    except AttributeError:
+    except AttributeError:  # pragma: no cover
         pass
     return Element("str:TextFormat", **attrib)
 
@@ -834,7 +834,7 @@ def _mdr(obj: model.MetadataReport):
     # TODO Write the id=… attribute
     elem = Element("md:Report")
 
-    if obj.target:
+    if obj.target:  # pragma: no cover
         elem.append(writer.recurse(obj.target))
     if obj.attaches_to:
         elem.append(writer.recurse(obj.attaches_to))
@@ -857,8 +857,10 @@ def _tok(obj: model.TargetObjectKey):
 @writer
 def _tov(obj: model.TargetObjectValue):
     if isinstance(obj.value_for, str):
+        # NB value_for should be MetadataAttribute, but currently not due to limitations
+        #    of .reader.xml.v21
         id_: str = obj.value_for
-    else:
+    else:  # pragma: no cover
         id_ = obj.value_for.id
 
     elem = Element("md:ReferenceValue", id=id_)
@@ -869,8 +871,8 @@ def _tov(obj: model.TargetObjectValue):
         elem.append(
             Element("md:ObjectReference", Element("URN", sdmx.urn.make(obj.obj)))
         )
-    else:
-        assert False
+    else:  # pragma: no cover
+        raise NotImplementedError(type(obj))
 
     return elem
 
@@ -884,7 +886,7 @@ def _ra(obj: model.ReportedAttribute):
         # NB value_for should be MetadataAttribute, but currently not due to limitations
         #    of .reader.xml.v21
         attrib.update(id=obj.value_for)
-    else:
+    else:  # pragma: no cover
         attrib.update(id=obj.value_for.id)
 
     if isinstance(obj, model.OtherNonEnumeratedAttributeValue):
@@ -894,7 +896,7 @@ def _ra(obj: model.ReportedAttribute):
             attrib.update(value=obj.value)
     elif isinstance(obj, model.XHTMLAttributeValue):
         child.append(Element("com:StructuredText", obj.value))
-    else:
+    else:  # pragma: no cover
         raise NotImplementedError
 
     if len(obj.child):

From e4d53d6bce9517f903c9c75834fb204cf26d0c81 Mon Sep 17 00:00:00 2001
From: Paul Natsuo Kishimoto <mail@paul.kishimoto.name>
Date: Tue, 3 Sep 2024 13:07:12 +0200
Subject: [PATCH 45/45] Add #192 to doc/whatsnew

---
 doc/whatsnew.rst  | 21 +++++++++++++++++++--
 sdmx/model/v21.py |  4 ++--
 sdmx/model/v30.py | 14 ++++++++------
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst
index 036ca9133..60ca60bc0 100644
--- a/doc/whatsnew.rst
+++ b/doc/whatsnew.rst
@@ -3,8 +3,25 @@
 What's new?
 ***********
 
-.. Next release
-.. ============
+Next release
+============
+
+- :class:`MetadataStructureDefinition <.BaseMetadataStructureDefinition>` and :class:`MetadataSet <.BaseMetadataSet>` can be written to and read from SDMX-ML (:pull:`192`).
+
+  - Clarify differences between :attr:`.v21.MetadataSet.structured_by` and :attr:`.v30.MetadataSet.structured_by`, according to the respective standards documents.
+  - Read and write :class:`.MetadataAttribute`, :class:`.MetadataReport`, :class:`.ReportedAttribute`, :class:`.Period`, and associated classes and subclasses.
+  - :class:`.XHTMLAttributeValue` contents are stored as :mod:`lxml.etree` nodes.
+  - MetadataStructureDefinition is included when writing :class:`.StructureMessage`.
+
+- Update base url for :ref:`WB_WDI` source to use HTTPS instead of plain HTTP (:issue:`191`, :pull:`192`).
+- Improvements to :mod:`.reader.xml` and :mod:`.reader.xml.v21` (:pull:`192`).
+
+  - Correctly associate :class:`.Item` in :class:`.ItemScheme` with its parent, even if the parent is defined after the child (“forward reference”).
+  - Bugfix: correctly handle a :class:`.MaintainableArtefact` that is explicitly included in a message (that is, not merely referenced), but with :py:`is_external_reference = True`; the value given in the file is preserved.
+  - Bugfix: :class:`.FacetValueType` is written in UpperCamelCase per the standard.
+    The standard specifies lowerCamelCase only in the Information Model.
+  - Bugfix: erroneous extra :xml:`<Ref ... style="Ref"/>` attribute is no longer written.
+- Expand logged information in :meth:`.ComponentList.compare` (:pull:`192`).
 
 v2.16.0 (2024-08-16)
 ====================
diff --git a/sdmx/model/v21.py b/sdmx/model/v21.py
index 8779c5734..261584579 100644
--- a/sdmx/model/v21.py
+++ b/sdmx/model/v21.py
@@ -514,8 +514,8 @@ class MetadataSet(NameableArtefact, common.BaseMetadataSet):
     described_by: Optional[MetadataflowDefinition] = None
     # described_by: Optional[ReportStructure] = None
 
-    #: .. seealso:: :attr:`.v30.MetadataSet.structured_by`, which has different
-    #:    semantics.
+    #: .. seealso::
+    #:    :attr:`.v30.MetadataSet.structured_by`, which has different semantics.
     structured_by: Optional[MetadataStructureDefinition] = None
 
     #: Analogous to :attr:`.v30.MetadataSet.provided_by`.
diff --git a/sdmx/model/v30.py b/sdmx/model/v30.py
index fa6d2f11c..707e39fbb 100644
--- a/sdmx/model/v30.py
+++ b/sdmx/model/v30.py
@@ -486,18 +486,20 @@ class MetadataSet(MaintainableArtefact, common.BaseMetadataSet):
     valid_to: Optional[str] = None
     set_id: Optional[str] = None
 
-    #: .. note:: According to the standard, MetadataSet has **two** associations, both
-    #:    named :py:`.described_by`: one to a :class:`.Metadataflow`, and the other to a
+    #: .. note::
+    #:    According to the standard, MetadataSet has **two** associations, both named
+    #:    :py:`.described_by`: one to a :class:`.Metadataflow`, and the other to a
     #:    :class:`.MetadataProvisionAgreement`. :mod:`sdmx` implements the first,
     #:    because it is consistent with SDMX 2.1.
     described_by: Optional[Metadataflow] = None
 
     # described_by: Optional[MetadataProvisionAgreement] = None
 
-    #: .. note:: According to the standard, this differs from
-    #:    :attr:`v21.MetadataSet.structured_by` in pointing to
-    #:    :attr:`.MetadataStructureDefinition.attributes`, directly, rather than to the
-    #:    MetadataStructureDefinition itself.
+    #: .. note::
+    #:    According to the standard, this differs from
+    #:    :attr:`v21.MetadataSet.structured_by` in that it points directly to
+    #:    :attr:`.MetadataStructureDefinition.attributes`, rather than to the
+    #:    MetadataStructureDefinition that contains the attribute descriptor.
     structured_by: Optional[MetadataAttributeDescriptor] = None
 
     #: Analogous to :attr:`.v21.MetadataSet.published_by`.