Update file hashing to read blocks instead of whole file

GoogleCloudPlatform · Oct 30, 2024 · 8ed8026 · 8ed8026
1 parent eacd650
commit 8ed8026
Showing 1 changed file with 9 additions and 4 deletions.
diff --git a/lib/ramble/ramble/util/hashing.py b/lib/ramble/ramble/util/hashing.py
@@ -10,13 +10,18 @@
 import hashlib
 import spack.util.spack_json as sjson
 
+BLOCK_SIZE = 1024 * 1024
+
 
 def hash_file(file_path):
-    file_hash = None
+    file_hash = hashlib.sha256()
     with open(file_path, "rb") as f:
-        bytes = f.read()
-        file_hash = hashlib.sha256(bytes).hexdigest()
-    return file_hash
+        while True:
+            bytes = f.read(BLOCK_SIZE)
+            if not bytes:
+                break
+            file_hash.update(bytes)
+    return file_hash.hexdigest()
 
 
 def hash_string(string):