[CDF-23957] 🧐 Robustify hash calculation

CHANGELOG.cdf-tk.md

@@ -15,6 +15,12 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## TBD
+
+### Fixed
+
+- Deploying functions containing files that are not `utf-8` encoded no longer raises a `UnicodeDecodeError`.
+
 ## [0.4.3] - 2025-02-04
 
 ### Fixed

cognite_toolkit/_cdf_tk/commands/build.py

@@ -468,7 +468,10 @@ def _replace_variables(
                 self.console(f"Processing file {source_path.name}...")
 
             content = safe_read(source_path)
-            source = SourceLocationEager(source_path, calculate_str_or_file_hash(content, shorten=True))
+            # We cannot use the content as the basis for hash as this have been encoded.
+            # Instead, we use the source path, which will hash the bytes of the file directly,
+            # which is what we do in the deploy step to verify that the source file has not changed.
+            source = SourceLocationEager(source_path, calculate_str_or_file_hash(source_path, shorten=True))
 
             content = variables.replace(content, source_path.suffix)
 

cognite_toolkit/_cdf_tk/utils/hashing.py

@@ -42,15 +42,11 @@ def calculate_secure_hash(item: dict[str, Any]) -> str:
 
 
 def calculate_str_or_file_hash(content: str | Path, shorten: bool = False) -> str:
-    sha256_hash = hashlib.sha256()
-    if isinstance(content, Path):
-        content = content.read_text("utf-8")
-    # Get rid of Windows line endings to make the hash consistent across platforms.
-    sha256_hash.update(content.encode("utf-8").replace(b"\r\n", b"\n"))
-    calculated = sha256_hash.hexdigest()
-    if shorten:
-        return calculated[:8]
-    return calculated
+    if isinstance(content, str):
+        byte_content = content.encode("utf-8")
+    else:
+        byte_content = content.read_bytes()
+    return calculate_bytes_or_file_hash(byte_content, shorten)
 
 
 def calculate_bytes_or_file_hash(content: bytes | Path, shorten: bool = False) -> str:

tests/test_unit/test_cdf_tk/test_commands/test_build.py

@@ -99,6 +99,7 @@ def test_build_valid_read_int_version(self) -> None:
         source_filepath = MagicMock(spec=Path)
         source_filepath.read_text.return_value = raw_yaml
         source_filepath.suffix = ".yaml"
+        source_filepath.read_bytes.return_value = raw_yaml.encode("utf-8")
 
         source_files = cmd._replace_variables(
             [source_filepath], BuildVariables([]), TransformationLoader.folder_name, Path("my_module"), verbose=False