fix: handle bytes() if payload has unicode (#152)

* fix: handle bytes() if payload has unicode

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* docs: revert suggestion to use bytes

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* feat: restore write_to_rfc822

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* refactor: MetadataPolicy -> RFC822Policy

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

README.md

@@ -30,7 +30,7 @@ metadata = StandardMetadata.from_pyproject(parsed_pyproject, allow_extra_keys =
 print(metadata.entrypoints)  # same fields as defined in PEP 621
 
 pkg_info = metadata.as_rfc822()
-print(bytes(pkg_info).decode("utf-8"))  # core metadata
+print(str(pkg_info)))  # core metadata
 ```
 
 ## METADATA 2.4

pyproject_metadata/__init__.py

@@ -67,11 +67,13 @@
     'ConfigurationError',
     'ConfigurationWarning',
     'License',
+    'RFC822Message',
+    'RFC822Policy',
     'Readme',
     'StandardMetadata',
-    'validate_top_level',
     'validate_build_system',
     'validate_project',
+    'validate_top_level',
 ]
 
 
@@ -126,22 +128,44 @@ class _SmartMessageSetter:
     If a value contains a newline, indent it (may produce a warning in the future).
     """
 
-    message: email.message.Message
+    message: email.message.EmailMessage
 
     def __setitem__(self, name: str, value: str | None) -> None:
         if not value:
             return
         self.message[name] = value
 
 
-class MetadataPolicy(email.policy.Compat32):
-    def fold(self, name: str, value: str) -> str:
+class RFC822Policy(email.policy.EmailPolicy):
+    """
+    This is `email.policy.EmailPolicy`, but with a simple ``header_store_parse``
+    implementation that handles multiline values, and some nice defaults.
+    """
+
+    utf8 = True
+    mangle_from_ = False
+    max_line_length = 0
+
+    def header_store_parse(self, name: str, value: str) -> tuple[str, str]:
         size = len(name) + 2
         value = value.replace('\n', '\n' + ' ' * size)
-        return f'{name}: {value}\n'
+        return (name, value)
 
-    def fold_binary(self, name: str, value: str) -> bytes:
-        return self.fold(name, value).encode('utf-8')
+
+class RFC822Message(email.message.EmailMessage):
+    """
+    This is `email.message.EmailMessage` with two small changes: it defaults to
+    our `RFC822Policy`, and it correctly writes unicode when being called
+    with `bytes()`.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(policy=RFC822Policy())
+
+    def as_bytes(
+        self, unixfrom: bool = False, policy: email.policy.Policy | None = None
+    ) -> bytes:
+        return self.as_string(unixfrom, policy=policy).encode('utf-8')
 
 
 class DataFetcher:
@@ -611,10 +635,14 @@ def __setattr__(self, name: str, value: Any) -> None:
             self._update_dynamic(value)
         super().__setattr__(name, value)
 
-    def as_rfc822(self) -> email.message.Message:  # noqa: C901
+    def as_rfc822(self) -> RFC822Message:
+        message = RFC822Message()
+        self.write_to_rfc822(message)
+        return message
+
+    def write_to_rfc822(self, message: email.message.EmailMessage) -> None:  # noqa: C901
         self.validate(warn=False)
 
-        message = email.message.Message(policy=MetadataPolicy())
         smart_message = _SmartMessageSetter(message)
 
         smart_message['Metadata-Version'] = self.metadata_version
@@ -675,8 +703,6 @@ def as_rfc822(self) -> email.message.Message:  # noqa: C901
                     raise ConfigurationError(msg)
                 smart_message['Dynamic'] = field
 
-        return message
-
     def _name_list(self, people: list[tuple[str, str | None]]) -> str:
         return ', '.join(name for name, email_ in people if not email_)
 

tests/test_rfc822.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import email.message
+import inspect
 import textwrap
 
 import pytest
@@ -39,6 +40,12 @@
             ],
             'Foo: Unicøde\n',
         ),
+        (
+            [
+                ('Foo', '🕵️'),
+            ],
+            'Foo: 🕵️\n',
+        ),
         # None
         (
             [
@@ -103,7 +110,7 @@
     ],
 )
 def test_headers(items: list[tuple[str, str]], data: str) -> None:
-    message = email.message.Message(policy=pyproject_metadata.MetadataPolicy())
+    message = pyproject_metadata.RFC822Message()
     smart_message = pyproject_metadata._SmartMessageSetter(message)
 
     for name, value in items:
@@ -119,14 +126,12 @@ def test_headers(items: list[tuple[str, str]], data: str) -> None:
 
 
 def test_body() -> None:
-    message = email.message.Message(policy=pyproject_metadata.MetadataPolicy())
+    message = pyproject_metadata.RFC822Message()
 
     message['ItemA'] = 'ValueA'
     message['ItemB'] = 'ValueB'
     message['ItemC'] = 'ValueC'
-
-    message.set_payload(
-        textwrap.dedent("""
+    body = inspect.cleandoc("""
         Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris congue semper
         fermentum. Nunc vitae tempor ante. Aenean aliquet posuere lacus non faucibus.
         In porttitor congue luctus. Vivamus eu dignissim orci. Donec egestas mi ac
@@ -139,33 +144,24 @@ def test_body() -> None:
         lacus blandit. Ut volutpat sollicitudin dapibus. Integer vitae lacinia ex, eget
         finibus nulla. Donec sit amet ante in neque pulvinar faucibus sed nec justo.
         Fusce hendrerit massa libero, sit amet pulvinar magna tempor quis. ø
-    """)
-    )
-
-    assert str(message) == textwrap.dedent("""\
+        """)
+    headers = inspect.cleandoc("""
         ItemA: ValueA
         ItemB: ValueB
         ItemC: ValueC
+        """)
+    full = f'{headers}\n\n{body}'
 
+    message.set_payload(textwrap.dedent(body))
 
-        Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris congue semper
-        fermentum. Nunc vitae tempor ante. Aenean aliquet posuere lacus non faucibus.
-        In porttitor congue luctus. Vivamus eu dignissim orci. Donec egestas mi ac
-        ipsum volutpat, vel elementum sapien consectetur. Praesent dictum finibus
-        fringilla. Sed vel feugiat leo. Nulla a pharetra augue, at tristique metus.
-
-        Aliquam fermentum elit at risus sagittis, vel pretium augue congue. Donec leo
-        risus, faucibus vel posuere efficitur, feugiat ut leo. Aliquam vestibulum vel
-        dolor id elementum. Ut bibendum nunc interdum neque interdum, vel tincidunt
-        lacus blandit. Ut volutpat sollicitudin dapibus. Integer vitae lacinia ex, eget
-        finibus nulla. Donec sit amet ante in neque pulvinar faucibus sed nec justo.
-        Fusce hendrerit massa libero, sit amet pulvinar magna tempor quis. ø
-    """)
+    assert str(message) == full
 
     new_message = email.message_from_string(str(message))
     assert new_message.items() == message.items()
     assert new_message.get_payload() == message.get_payload()
 
+    assert bytes(message) == full.encode('utf-8')
+
 
 def test_convert_optional_dependencies() -> None:
     metadata = pyproject_metadata.StandardMetadata.from_pyproject(