Update override validation to sort files prior to hashing.

BaseTools/Plugin/OverrideValidation/OverrideValidation.py

@@ -566,8 +566,11 @@ def get_dsc_inf_list(self, thebuilder):
 except ImportError:
     pass
 
-# This calculates the md5 for the inf file as well as all the first order include source files
-# path: the absolute path to the module's inf file
+
+# This calculates the md5 for the specified path.
+# If the path is an INF, it hashes the sources section
+# If its a directory, all the files in the directory and
+# sub directories will be hashed.
 def ModuleHashCal(path):
 
     sourcefileList = []
@@ -578,10 +581,10 @@ def ModuleHashCal(path):
     folderpath = os.path.dirname(path)
 
     if os.path.isdir(path):
-      # Collect all files in this folder to the list
-      for subdir, _, files in os.walk(path):
-          for file in files:
-              sourcefileList.append(os.path.join(subdir, file))
+        # Collect all files in this folder to the list
+        for subdir, _, files in os.walk(path):
+            for file in files:
+                sourcefileList.append(os.path.join(subdir, file))
     else:
         sourcefileList.append(path)
 
@@ -613,6 +616,67 @@ def ModuleHashCal(path):
     result = hash_obj.hexdigest()
     return result
 
+
+# This calculates the md5 for the specified path.
+# If the path is an INF, it hashes the sources section
+# If its a directory, all the files in the directory and
+# sub directories will be hashed.
+# this version will sort the files prior to generating the hashes
+# due to some errors encountered under linux where the file order
+# returned from os.walk were different, causing a hash mismatch
+def ModuleHash2Cal(path):
+
+    sourcefileList = []
+    binaryfileList = []
+    hash_obj = hashlib.md5()
+
+    # Find the specific line of Sources section
+    folderpath = os.path.dirname(path)
+
+    if os.path.isdir(path):
+        # Collect all files in this folder to the list
+        for subdir, _, files in os.walk(path):
+            for file in files:
+                sourcefileList.append(os.path.join(subdir, file))
+    else:
+        sourcefileList.append(path)
+
+    if path.lower().endswith(".inf") and os.path.isfile(path):
+
+        # Use InfParser to parse sources section
+        ip = InfParser()
+        ip.ParseFile(path)
+
+        # Add all referenced source files in addition to our inf file list
+        for source in ip.Sources:
+            sourcefileList.append(os.path.normpath(os.path.join(folderpath, source)))
+
+        # Add all referenced binary files to our binary file list
+        for binary in ip.Binaries:
+            binaryfileList.append(os.path.normpath(os.path.join(folderpath, binary)))
+
+    # Convert to '/' path separator prior to sorting to avoid ordering mismatches
+    sourcefileList = [sfile.replace("\\", "/") for sfile in sourcefileList]
+    binaryfileList = [bfile.replace("\\", "/") for bfile in binaryfileList]
+
+    sourcefileList.sort()
+    binaryfileList.sort()
+
+    for sfile in sourcefileList:
+        # print('Calculated: %s' %(sfile)) #Debug only
+        with open(os.path.normpath(sfile), 'rb') as entry:
+            # replace \r\n with \n to take care of line terminators
+            hash_obj.update(entry.read().replace(b'\r\n', b'\n'))
+
+    for bfile in binaryfileList:
+        # print('Calculated: %s' %(bfile)) #Debug only
+        with open(os.path.normpath(bfile), 'rb') as entry:
+            hash_obj.update(entry.read())
+
+    result = hash_obj.hexdigest()
+    return result
+
+
 def ModuleGitPatch(path, git_hash):
     ''' return a git patch of the given file since the hash '''
     GitOutput = io.StringIO()
@@ -767,15 +831,16 @@ def path_parse():
                 abs_path = pathtool.GetAbsolutePathOnThisSystemFromEdk2RelativePath(rel_path)
                 if abs_path is not None:
                     mod_hash = ModuleHashCal(abs_path)
+                    mod_hash2 = ModuleHash2Cal(abs_path)
                     # only update the line if the hash has changed - this ensures the timestamp tracks actual changes rather than last time it was run.
-                    if (mod_hash != match.group(4)):
+                    if (mod_hash != match.group(4) or mod_hash2 != match.group(4)):
                         VERSION_INDEX = Paths.Version - 1
 
                         if VERSION_INDEX == 0:
-                            line = '#%s : %08d | %s | %s | %s\n' % (match.group(1), OVERRIDE_FORMAT_VERSION_1[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime(TIMESTAMP_FORMAT))
+                            line = '#%s : %08d | %s | %s | %s\n' % (match.group(1), OVERRIDE_FORMAT_VERSION_1[0], rel_path, mod_hash2, datetime.now(timezone.utc).strftime(TIMESTAMP_FORMAT))
                         elif VERSION_INDEX == 1:
                             git_hash = ModuleGitHash(abs_path)
-                            line = '#%s : %08d | %s | %s | %s | %s\n' % (match.group(1), OVERRIDE_FORMAT_VERSION_2[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime(TIMESTAMP_FORMAT), git_hash)
+                            line = '#%s : %08d | %s | %s | %s | %s\n' % (match.group(1), OVERRIDE_FORMAT_VERSION_2[0], rel_path, mod_hash2, datetime.now(timezone.utc).strftime(TIMESTAMP_FORMAT), git_hash)
                         print("Updating:\n" + line)
                 else:
                     print(f"Warning: Could not resolve relative path {rel_path}. Override line not updated.\n")
@@ -825,7 +890,7 @@ def path_parse():
                     sys.exit(1)
 
             rel_path = rel_path.replace('\\', '/')
-            mod_hash = ModuleHashCal(Paths.TargetPath)
+            mod_hash = ModuleHash2Cal(Paths.TargetPath)
 
             VERSION_INDEX = Paths.Version - 1