wip

cloudinit/url_helper.py

@@ -143,7 +143,8 @@ def read_ftps(url: str, timeout: float = 5.0, **kwargs: dict) -> "FtpResponse":
             ftp_tls.prot_p()
             LOG.debug("Reading file: %s", url_parts.path)
             ftp_tls.retrbinary(f"RETR {url_parts.path}", callback=buffer.write)
-            response = FtpResponse(url_parts.path, contents=buffer)
+
+            response = FtpResponse(buffer.getvalue())
             LOG.debug("Closing connection")
             ftp_tls.close()
             return response
@@ -176,7 +177,7 @@ def read_ftps(url: str, timeout: float = 5.0, **kwargs: dict) -> "FtpResponse":
             )
             LOG.debug("Reading file: %s", url_parts.path)
             ftp.retrbinary(f"RETR {url_parts.path}", callback=buffer.write)
-            response = FtpResponse(url_parts.path, contents=buffer)
+            response = FtpResponse(contents=buffer.getvalue())
             LOG.debug("Closing connection")
             ftp.close()
             return response
@@ -191,7 +192,7 @@ def _read_file(path: str, **kwargs) -> "FileResponse":
         LOG.warning("Unable to post data to file resource %s", path)
     try:
         contents = util.load_binary_file(path)
-        return FileResponse(path, contents=contents)
+        return FileResponse(contents, path)
     except FileNotFoundError as e:
         raise UrlError(cause=e, code=NOT_FOUND, headers=None, url=path) from e
     except IOError as e:
@@ -226,11 +227,11 @@ def read_file_or_url(
 # read_file_or_url can return this or that object and the
 # 'user' of those objects will not need to know the difference.
 class StringResponse:
-    def __init__(self, contents, code=200):
+    def __init__(self, contents, url, code=200):
         self.code = code
         self.headers = {}
         self.contents = contents
-        self.url = None
+        self.url = url
 
     def ok(self, *args, **kwargs):
         if self.code != 200:
@@ -242,14 +243,13 @@ def __str__(self):
 
 
 class FileResponse(StringResponse):
-    def __init__(self, path, contents, code=200):
-        StringResponse.__init__(self, contents, code=code)
+    def __init__(self, contents: bytes, url: str, code=200):
+        super().__init__(contents, url, code=code)
 
 
 class FtpResponse(StringResponse):
-    def __init__(self, path, contents):
-        super().__init__(self, contents)
-        self.url = path
+    def __init__(self, contents: bytes, url: str):
+        super().__init__(contents, url)
 
 
 class UrlResponse:
@@ -263,10 +263,10 @@ def contents(self) -> bytes:
         return self._response.content
 
     @property
-    def url(self):
+    def url(self) -> str:
         return self._response.url
 
-    def ok(self, redirects_ok=False):
+    def ok(self, redirects_ok=False) -> bool:
         upper = 300
         if redirects_ok:
             upper = 400
@@ -276,11 +276,11 @@ def ok(self, redirects_ok=False):
             return False
 
     @property
-    def headers(self):
+    def headers(self) -> str:
         return self._response.headers
 
     @property
-    def code(self):
+    def code(self) -> int:
         return self._response.status_code
 
     def __str__(self):

tests/integration_tests/datasources/test_nocloud.py

@@ -371,7 +371,7 @@ def _boot_with_cmdline(
                 """
             ),
         )
-        client.write_to_file("/vendor-data", "\n")
+        client.write_to_file("/vendor-data", "")
 
         # set the kernel commandline, reboot with it
         override_kernel_cmdline(cmdline, client)
@@ -387,6 +387,7 @@ def test_nocloud_ftp_unencrypted_server_succeeds(
         cmdline = "ds=nocloud;seedfrom=ftp://0.0.0.0:2121"
         self._boot_with_cmdline(cmdline, client)
         verify_clean_boot(client, ignore_warnings=self.expected_warnings)
+        assert "ftp-instance" == client.execute("hostname").rstrip()
 
     def test_nocloud_ftps_unencrypted_server_fails(
         self, client: IntegrationInstance
@@ -419,6 +420,7 @@ def test_nocloud_ftps_encrypted_server_succeeds(
         cmdline = "ds=nocloud;seedfrom=ftps://localhost:2121"
         self._boot_with_cmdline(cmdline, client, encrypted=True)
         verify_clean_boot(client, ignore_warnings=self.expected_warnings)
+        assert "ftp-instance" == client.execute("hostname").rstrip()
 
     def test_nocloud_ftp_encrypted_server_fails(
         self, client: IntegrationInstance

tests/integration_tests/util.py

@@ -45,7 +45,7 @@ def verify_ordered_items_in_text(to_verify: list, text: str):
         index = matched.start()
 
 
-def format_found(header: str, items: list) -> str:
+def _format_found(header: str, items: list) -> str:
     """Helper function to format assertion message"""
 
     # do nothing, allows this formatter to be "stackable"
@@ -154,16 +154,16 @@ def verify_clean_boot(
             message += "Unexpected warnings or errors found"
 
         # errors are probably more important, order them first
-        message += format_found(
+        message += _format_found(
             "Found unexpected errors", list(unexpected_errors)
         )
-        message += format_found(
+        message += _format_found(
             "Required errors not found", list(required_errors_not_found)
         )
-        message += format_found(
+        message += _format_found(
             "Found unexpected warnings", list(unexpected_warnings)
         )
-        message += format_found(
+        message += _format_found(
             "Required warnings not found", list(required_warnings_not_found)
         )
         assert not errors, message

tests/unittests/sources/test_maas.py

@@ -116,7 +116,7 @@ def my_readurl(*args, **kwargs):
             short = url[len(prefix) :]
             if short not in data:
                 raise url_helper.UrlError("not found", code=404, url=url)
-            return url_helper.StringResponse(data[short])
+            return url_helper.StringResponse(data[short], url)
 
         # Now do the actual call of the code under test.
         with mock.patch("cloudinit.url_helper.readurl") as mock_readurl:

tests/unittests/test__init__.py

@@ -252,7 +252,7 @@ def test_invalid_content(self, m_read, tmpdir):
         key = "cloud-config-url"
         url = "http://example.com/foo"
         cmdline = "ro %s=%s bar=1" % (key, url)
-        m_read.return_value = url_helper.StringResponse(b"unexpected blob")
+        m_read.return_value = url_helper.StringResponse(b"unexpected blob", "http://example.com/")
 
         fpath = tmpdir.join("ccfile")
         lvl, msg = main.attempt_cmdline_url(
@@ -288,7 +288,7 @@ def test_valid_content(self, m_read, tmpdir):
         payload = b"#cloud-config\nmydata: foo\nbar: wark\n"
         cmdline = "ro %s=%s bar=1" % ("cloud-config-url", url)
 
-        m_read.return_value = url_helper.StringResponse(payload)
+        m_read.return_value = url_helper.StringResponse(payload, "http://example.com")
         fpath = tmpdir.join("ccfile")
         lvl, msg = main.attempt_cmdline_url(
             fpath, network=True, cmdline=cmdline

tests/unittests/test_util.py

@@ -2529,7 +2529,7 @@ def fake_response(url, timeout, retries):
                 else:
                     _url, _, md_type = parsed_url.netloc.partition("8008")
                 path = f"/{md_type}"
-            return url_helper.StringResponse(f"{path}: 1")
+            return url_helper.StringResponse(f"{path}: 1", "http://url/")
 
         m_read.side_effect = fake_response