[mtouch/mmp] Fix tracking of whether the static registrar should run again or not. Fixes #641.

tests/mtouch/MTouch.cs

@@ -363,6 +363,21 @@ public void RebuildTest_DontLink ()
 			}
 		}
 
+		void DumpFileStats (MTouchTool mtouch)
+		{
+			if (mtouch.Verbosity < 1)
+				return;
+			var directory = mtouch.Cache;
+			var files = Directory.GetFileSystemEntries (directory, "*", SearchOption.AllDirectories).ToList ();
+			files.Sort ((string x, string y) => string.CompareOrdinal (x, y));
+			var max = files.Max ((v) => v.Length);
+
+			var format = "    {0,-" + max + "} {1}";
+			foreach (var file in  files) {
+				Console.WriteLine (format, file, File.GetLastWriteTimeUtc (file).ToString ("HH:mm:ss.fffffff"));
+			}
+		}
+
 		[Test]
 		[TestCase ("single", "", false, new string [] { } )]
 		[TestCase ("dual", "armv7,arm64", false, new string [] { })]
@@ -397,17 +412,20 @@ public void RebuildTest_WithExtensions (string name, string abi, bool debug, str
 					mtouch.DSym = false; // faster test
 					mtouch.MSym = false; // faster test
 					mtouch.NoStrip = true; // faster test
+					//mtouch.Verbosity = 20; // Set the mtouch verbosity to something to print the mtouch output to the terminal. This will also enable additional debug output.
 
 					var timestamp = DateTime.MinValue;
 
 					mtouch.AssertExecute (MTouchAction.BuildDev, "first build");
 					Console.WriteLine ($"{DateTime.Now} **** FIRST BUILD DONE ****");
+					DumpFileStats (mtouch);
 
 					timestamp = DateTime.Now;
 					System.Threading.Thread.Sleep (1000); // make sure all new timestamps are at least a second older. HFS+ has a 1s timestamp resolution :(
 
 					mtouch.AssertExecute (MTouchAction.BuildDev, "second build");
 					Console.WriteLine ($"{DateTime.Now} **** SECOND BUILD DONE ****");
+					DumpFileStats (mtouch);
 
 					mtouch.AssertNoneModified (timestamp, name);
 					extension.AssertNoneModified (timestamp, name);
@@ -416,13 +434,15 @@ public void RebuildTest_WithExtensions (string name, string abi, bool debug, str
 					new FileInfo (extension.RootAssembly).LastWriteTimeUtc = DateTime.UtcNow;
 					mtouch.AssertExecute (MTouchAction.BuildDev, "touch extension executable");
 					Console.WriteLine ($"{DateTime.Now} **** TOUCH EXTENSION EXECUTABLE DONE ****");
+					DumpFileStats (mtouch);
 					mtouch.AssertNoneModified (timestamp, name);
 					extension.AssertNoneModified (timestamp, name);
 
 					// Touch the main app's executable, nothing should change
 					new FileInfo (mtouch.RootAssembly).LastWriteTimeUtc = DateTime.UtcNow;
 					mtouch.AssertExecute (MTouchAction.BuildDev, "touch main app executable");
 					Console.WriteLine ($"{DateTime.Now} **** TOUCH MAIN APP EXECUTABLE DONE ****");
+					DumpFileStats (mtouch);
 					mtouch.AssertNoneModified (timestamp, name);
 					extension.AssertNoneModified (timestamp, name);
 
@@ -440,6 +460,7 @@ public void RebuildTest_WithExtensions (string name, string abi, bool debug, str
 					extension.CreateTemporaryServiceExtension (extraCode: codeB);
 					mtouch.AssertExecute (MTouchAction.BuildDev, "change extension executable");
 					Console.WriteLine ($"{DateTime.Now} **** CHANGE EXTENSION EXECUTABLE DONE ****");
+					DumpFileStats (mtouch);
 					mtouch.AssertNoneModified (timestamp, name);
 					extension.AssertNoneModified (timestamp, name, "testServiceExtension", "testServiceExtension.aotdata.armv7", "testServiceExtension.aotdata.arm64", "testServiceExtension.dll");
 
@@ -450,6 +471,7 @@ public void RebuildTest_WithExtensions (string name, string abi, bool debug, str
 					mtouch.CreateTemporaryApp (extraCode: codeB);
 					mtouch.AssertExecute (MTouchAction.BuildDev, "change app executable");
 					Console.WriteLine ($"{DateTime.Now} **** CHANGE APP EXECUTABLE DONE ****");
+					DumpFileStats (mtouch);
 					mtouch.AssertNoneModified (timestamp, name, "testApp", "testApp.aotdata.armv7", "testApp.aotdata.arm64", "testApp.exe");
 					extension.AssertNoneModified (timestamp, name);
 
@@ -460,6 +482,7 @@ public void RebuildTest_WithExtensions (string name, string abi, bool debug, str
 					File.WriteAllText (extension.RootAssembly + ".config", "<configuration></configuration>");
 					mtouch.AssertExecute (MTouchAction.BuildDev, "add config to extension dll");
 					Console.WriteLine ($"{DateTime.Now} **** ADD CONFIG TO EXTENSION DONE ****");
+					DumpFileStats (mtouch);
 					mtouch.AssertNoneModified (timestamp, name);
 					extension.AssertNoneModified (timestamp, name, "testServiceExtension.dll.config", "testServiceExtension", "testServiceExtension.aotdata.armv7", "testServiceExtension.aotdata.arm64");
 					CollectionAssert.Contains (Directory.EnumerateFiles (extension.AppPath, "*", SearchOption.AllDirectories).Select ((v) => Path.GetFileName (v)), "testServiceExtension.dll.config", "extension config added");
@@ -471,18 +494,19 @@ public void RebuildTest_WithExtensions (string name, string abi, bool debug, str
 					File.WriteAllText (mtouch.RootAssembly + ".config", "<configuration></configuration>");
 					mtouch.AssertExecute (MTouchAction.BuildDev, "add config to container exe");
 					Console.WriteLine ($"{DateTime.Now} **** ADD CONFIG TO CONTAINER DONE ****");
+					DumpFileStats (mtouch);
 					mtouch.AssertNoneModified (timestamp, name, "testApp.exe.config", "testApp", "testApp.aotdata.armv7", "testApp.aotdata.arm64");
 					extension.AssertNoneModified (timestamp, name);
 					CollectionAssert.Contains (Directory.EnumerateFiles (mtouch.AppPath, "*", SearchOption.AllDirectories).Select ((v) => Path.GetFileName (v)), "testApp.exe.config", "container config added");
 
 					timestamp = DateTime.Now;
 					System.Threading.Thread.Sleep (1000); // make sure all new timestamps are at least a second older. HFS+ has a 1s timestamp resolution :(
-
 					{
 						// Add a satellite to the extension.
 						var satellite = extension.CreateTemporarySatelliteAssembly ();
 						mtouch.AssertExecute (MTouchAction.BuildDev, "add satellite to extension");
 						Console.WriteLine ($"{DateTime.Now} **** ADD SATELLITE TO EXTENSION DONE ****");
+						DumpFileStats (mtouch);
 						mtouch.AssertNoneModified (timestamp, name, Path.GetFileName (satellite));
 						extension.AssertNoneModified (timestamp, name, Path.GetFileName (satellite));
 						extension.AssertModified (timestamp, name, Path.GetFileName (satellite));
@@ -497,6 +521,7 @@ public void RebuildTest_WithExtensions (string name, string abi, bool debug, str
 						var satellite = mtouch.CreateTemporarySatelliteAssembly ();
 						mtouch.AssertExecute (MTouchAction.BuildDev, "add satellite to container");
 						Console.WriteLine ($"{DateTime.Now} **** ADD SATELLITE TO CONTAINER DONE ****");
+						DumpFileStats (mtouch);
 						mtouch.AssertNoneModified (timestamp, name, Path.GetFileName (satellite));
 						extension.AssertNoneModified (timestamp, name, Path.GetFileName (satellite));
 						mtouch.AssertModified (timestamp, name, Path.GetFileName (satellite));

tools/common/Application.cs

@@ -126,7 +126,13 @@ public string PlatformName {
 			}
 		}
 
-		public static bool IsUptodate (string source, string target, bool check_contents = false)
+		// Checks if the source file has a time stamp later than the target file.
+		//
+		// Optionally check if the contents of the files are different after checking the timestamp.
+		//
+		// If check_stamp is true, the function will use the timestamp of a "target".stamp file
+		// if it's later than the timestamp of the "target" file itself.
+		public static bool IsUptodate (string source, string target, bool check_contents = false, bool check_stamp = true)
 		{
 			if (Driver.Force)
 				return false;
@@ -138,6 +144,14 @@ public static bool IsUptodate (string source, string target, bool check_contents
 				return false;
 			}
 
+			if (check_stamp) {
+				var tfi_stamp = new FileInfo (target + ".stamp");
+				if (tfi_stamp.Exists && tfi_stamp.LastWriteTimeUtc > tfi.LastWriteTimeUtc) {
+					Driver.Log (3, "Target '{0}' has a stamp file with newer timestamp ({1} > {2}), using the stamp file's timestamp", target, tfi_stamp.LastWriteTimeUtc, tfi.LastWriteTimeUtc);
+					tfi = tfi_stamp;
+				}
+			}
+
 			var sfi = new FileInfo (source);
 
 			if (sfi.LastWriteTimeUtc <= tfi.LastWriteTimeUtc) {
@@ -323,7 +337,10 @@ public static void UpdateFile (string source, string target, bool check_contents
 		}
 
 		// Checks if any of the source files have a time stamp later than any of the target files.
-		public static bool IsUptodate (IEnumerable<string> sources, IEnumerable<string> targets)
+		//
+		// If check_stamp is true, the function will use the timestamp of a "target".stamp file
+		// if it's later than the timestamp of the "target" file itself.
+		public static bool IsUptodate (IEnumerable<string> sources, IEnumerable<string> targets, bool check_stamp = true)
 		{
 			if (Driver.Force)
 				return false;
@@ -356,6 +373,14 @@ public static bool IsUptodate (IEnumerable<string> sources, IEnumerable<string>
 					return false;
 				}
 
+				if (check_stamp) {
+					var tfi_stamp = new FileInfo (t + ".stamp");
+					if (tfi_stamp.Exists && tfi_stamp.LastWriteTimeUtc > tfi.LastWriteTimeUtc) {
+						Driver.Log (3, "Target '{0}' has a stamp file with newer timestamp ({1} > {2}), using the stamp file's timestamp", t, tfi_stamp.LastWriteTimeUtc, tfi.LastWriteTimeUtc);
+						tfi = tfi_stamp;
+					}
+				}
+
 				var lwt = tfi.LastWriteTimeUtc;
 				if (max_source > lwt) {
 					Driver.Log (3, "Prerequisite '{0}' is newer than target '{1}' ({2} vs {3}).", max_s, t, max_source, lwt);

tools/common/Driver.cs

@@ -7,6 +7,7 @@
  */
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -304,7 +305,7 @@ static void FileMove (string source, string target)
 			File.Move (source, target);
 		}
 
-		static void MoveIfDifferent (string path, string tmp)
+		static void MoveIfDifferent (string path, string tmp, bool use_stamp = false)
 		{
 			// Don't read the entire file into memory, it can be quite big in certain cases.
 
@@ -325,10 +326,12 @@ static void MoveIfDifferent (string path, string tmp)
 				FileMove (tmp, path);
 			} else {
 				Log (3, "Target {0} is up-to-date.", path);
+				if (use_stamp)
+					Driver.Touch (path + ".stamp");
 			}
 		}
 
-		public static void WriteIfDifferent (string path, string contents)
+		public static void WriteIfDifferent (string path, string contents, bool use_stamp = false)
 		{
 			var tmp = path + ".tmp";
 
@@ -341,7 +344,7 @@ public static void WriteIfDifferent (string path, string contents)
 				}
 
 				File.WriteAllText (tmp, contents);
-				MoveIfDifferent (path, tmp);
+				MoveIfDifferent (path, tmp, use_stamp);
 			} catch (Exception e) {
 				File.WriteAllText (path, contents);
 				ErrorHelper.Warning (1014, e, "Failed to re-use cached version of '{0}': {1}.", path, e.Message);
@@ -350,7 +353,7 @@ public static void WriteIfDifferent (string path, string contents)
 			}
 		}
 
-		public static void WriteIfDifferent (string path, byte[] contents)
+		public static void WriteIfDifferent (string path, byte[] contents, bool use_stamp = false)
 		{
 			var tmp = path + ".tmp";
 
@@ -362,7 +365,7 @@ public static void WriteIfDifferent (string path, byte[] contents)
 				}
 
 				File.WriteAllBytes (tmp, contents);
-				MoveIfDifferent (path, tmp);
+				MoveIfDifferent (path, tmp, use_stamp);
 			} catch (Exception e) {
 				File.WriteAllBytes (path, contents);
 				ErrorHelper.Warning (1014, e, "Failed to re-use cached version of '{0}': {1}.", path, e.Message);
@@ -438,5 +441,30 @@ static void LogArguments (string [] arguments, int indentation)
 				}
 			}
 		}
+
+		public static void Touch (IEnumerable<string> filenames, DateTime? timestamp = null)
+		{
+			if (timestamp == null)
+				timestamp = DateTime.Now;
+			foreach (var filename in filenames) {
+				try {
+					var fi = new FileInfo (filename);
+					if (!fi.Exists) {
+						using (var fo = fi.OpenWrite ()) {
+							// Create an empty file.
+						}
+					}
+					fi.LastWriteTime = timestamp.Value;
+				} catch (Exception e) {
+					ErrorHelper.Warning (128, "Could not touch the file '{0}': {1}", filename, e.Message);
+				}
+			}
+		}
+
+		public static void Touch (params string [] filenames)
+		{
+			Touch ((IEnumerable<string>) filenames);
+		}
+
 	}
 }

tools/common/PInvokeWrapperGenerator.cs

@@ -63,8 +63,8 @@ public void End ()
 
 			Registrar.GeneratePInvokeWrappersEnd ();
 
-			Driver.WriteIfDifferent (HeaderPath, hdr.ToString () + "\n" + decls.ToString () + "\n" + ifaces.ToString () + "\n") ;
-			Driver.WriteIfDifferent (SourcePath, mthds.ToString () + "\n" + sb.ToString () + "\n");
+			Driver.WriteIfDifferent (HeaderPath, hdr.ToString () + "\n" + decls.ToString () + "\n" + ifaces.ToString () + "\n", true);
+			Driver.WriteIfDifferent (SourcePath, mthds.ToString () + "\n" + sb.ToString () + "\n", true);
 		}
 
 		public void ProcessMethod (MethodDefinition method)

tools/common/StaticRegistrar.cs

@@ -4508,6 +4508,9 @@ public void GenerateSingleAssembly (IEnumerable<AssemblyDefinition> assemblies,
 
 		public void Generate (IEnumerable<AssemblyDefinition> assemblies, string header_path, string source_path)
 		{
+			if (Target?.CachedLink == true)
+				throw ErrorHelper.CreateError (99, "Internal error: the static registrar should not execute unless the linker also executed (or was disabled). A potential workaround is to pass '-f' as an additional " + Driver.NAME + " argument to force a full build. Please file a bug report with a test case (https://bugzilla.xamarin.com).");
+
 			this.input_assemblies = assemblies;
 
 			foreach (var assembly in assemblies) {
@@ -4561,12 +4564,12 @@ void Generate (string header_path, string source_path)
 
 			FlushTrace ();
 
-			Driver.WriteIfDifferent (source_path, methods.ToString ());
+			Driver.WriteIfDifferent (source_path, methods.ToString (), true);
 
 			header.AppendLine ();
 			header.AppendLine (declarations);
 			header.AppendLine (interfaces);
-			Driver.WriteIfDifferent (header_path, header.ToString ());
+			Driver.WriteIfDifferent (header_path, header.ToString (), true);
 
 			header.Dispose ();
 			header = null;

tools/common/Target.cs

@@ -62,6 +62,12 @@ public Target (Application app)
 			this.StaticRegistrar = new StaticRegistrar (this);
 		}
 
+		public bool CachedLink {
+			get {
+				return cached_link;
+			}
+		}
+
 		public void ExtractNativeLinkInfo (List<Exception> exceptions)
 		{
 			foreach (var a in Assemblies) {
@@ -396,7 +402,7 @@ internal string GenerateReferencingSource (string reference_m, IEnumerable<Symbo
 			sb.AppendLine ("}");
 			sb.AppendLine ();
 
-			Driver.WriteIfDifferent (reference_m, sb.ToString ());
+			Driver.WriteIfDifferent (reference_m, sb.ToString (), true);
 
 #if MTOUCH
 			foreach (var abi in GetArchitectures (AssemblyBuildTarget.StaticObject)) {

tools/mmp/.gitignore

@@ -2,3 +2,5 @@ mmp
 temp-dir-mmp
 Xamarin.Mac.registrar.*
 SdkVersions.cs
+*.stamp
+

tools/mmp/driver.cs

@@ -68,6 +68,7 @@ enum Action {
 	}
 
 	public static partial class Driver {
+		internal const string NAME = "mmp";
 		internal static Application App = new Application (Environment.GetCommandLineArgs ());
 		static Target BuildTarget = new Target (App);
 		static List<string> references = new List<string> ();

tools/mtouch/.gitignore

@@ -7,3 +7,5 @@ SdkVersions.cs
 simlauncher-sgen
 simlauncher32-sgen
 simlauncher64-sgen
+*.stamp
+

tools/mtouch/Target.cs

@@ -537,17 +537,38 @@ public void ManagedLink ()
 			if (!Driver.Force) {
 				if (File.Exists (cache_path)) {
 					using (var reader = new StreamReader (cache_path)) {
-						string line;
+						string line = null;
+
 						while ((line = reader.ReadLine ()) != null) {
 							var colon = line.IndexOf (':');
 							if (colon == -1)
 								continue;
-							var appex = line.Substring (0, colon);
-							var asm = line.Substring (colon + 1);
-							List<string> asms;
-							if (!cached_output.TryGetValue (appex, out asms))
-								cached_output [appex] = asms = new List<string> ();
-							asms.Add (asm);
+							var key = line.Substring (0, colon);
+							var value = line.Substring (colon + 1);
+							switch (key) {
+							case "RemoveDynamicRegistrar":
+								switch (value) {
+								case "true":
+									App.Optimizations.RemoveDynamicRegistrar = true;
+									break;
+								case "false":
+									App.Optimizations.RemoveDynamicRegistrar = false;
+									break;
+								default:
+									App.Optimizations.RemoveDynamicRegistrar = null;
+									break;
+								}
+								Driver.Log (1, $"Optimization dynamic registrar removal loaded from cached results: {(App.Optimizations.RemoveDynamicRegistrar.HasValue ? (App.Optimizations.RemoveUIThreadChecks.Value ? "enabled" : "disabled") : "not set")}");
+								break;
+							default:
+								// key: app(ex)
+								// value: assembly
+								List<string> asms;
+								if (!cached_output.TryGetValue (key, out asms))
+									cached_output [key] = asms = new List<string> ();
+								asms.Add (value);
+								break;
+							}
 						}
 					}
 
@@ -683,6 +704,8 @@ public void ManagedLink ()
 
 			// Write the input files to the cache
 			using (var writer = new StreamWriter (cache_path, false)) {
+				var opt = App.Optimizations.RemoveDynamicRegistrar;
+				writer.WriteLine ($"RemoveDynamicRegistrar:{(opt.HasValue ? (opt.Value ? "true" : "false") : string.Empty)}");
 				foreach (var target in allTargets) {
 					foreach (var asm in target.Assemblies) {
 						writer.WriteLine ($"{target.App.AppDirectory}:{asm.FullPath}");