Add support for symlink files embedding to binlog

documentation/wiki/Building-Testing-and-Debugging-on-.Net-Core-MSBuild.md

@@ -6,6 +6,10 @@ MSBuild can be successfully built on Windows, OS X 10.13, Ubuntu 14.04, and Ubun
 
 `build.cmd -msbuildEngine dotnet`
 
+## Tests
+
+Follow [Running Unit Tests](Building-Testing-and-Debugging-on-.Net-Core-MSBuild.md#running-unit-tests) section of the developer guide chapter for .NET Framework
+
 # Unix
 
 ## The easy way

documentation/wiki/Building-Testing-and-Debugging-on-Full-Framework-MSBuild.md

@@ -25,6 +25,10 @@ To run the unit tests from Visual Studio:
 
 To build MSBuild and run all unit tests from the command line, use `.\build.cmd -test`.
 
+Some tests are creating symlinks to test associated functionality - in order for them to succeed you have two options:
+* Enable [Development Mode](https://learn.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development) on your machine.
+* Or run those tests elevated
+
 To mimic our CI job use `eng\CIBuild.cmd`. Be aware that this command may delete your local NuGet cache.
 
 The CI does two builds. In the second build, it uses the binaries from the first build to build the repository again.

src/Build.UnitTests/BinaryLogger_Tests.cs

@@ -207,6 +207,62 @@ public void BinaryLoggerShouldEmbedFilesViaTaskOutput()
             zipArchive.Entries.ShouldContain(zE => zE.Name.EndsWith("testtaskoutputfile.txt"));
         }
 
+        [Fact]
+        public void BinaryLoggerShouldEmbedSymlinkFilesViaTaskOutput()
+        {
+            string testFileName = "foobar.txt";
+            string symlinkName = "symlink1.txt";
+            string symlinkLvl2Name = "symlink2.txt";
+            TransientTestFolder testFolder = _env.DefaultTestDirectory.CreateDirectory("TestDir");
+            TransientTestFolder testFolder2 = _env.DefaultTestDirectory.CreateDirectory("TestDir2");
+            TransientTestFile testFile = testFolder.CreateFile(testFileName, string.Join(Environment.NewLine, new[] { "123", "456" }));
+            string symlinkPath = Path.Combine(testFolder2.Path, symlinkName);
+            string symlinkLvl2Path = Path.Combine(testFolder2.Path, symlinkLvl2Name);
+
+            string errorMessage = string.Empty;
+            Assert.True(NativeMethodsShared.MakeSymbolicLink(symlinkPath, testFile.Path, ref errorMessage), errorMessage);
+            Assert.True(NativeMethodsShared.MakeSymbolicLink(symlinkLvl2Path, symlinkPath, ref errorMessage), errorMessage);
+
+            using var buildManager = new BuildManager();
+            var binaryLogger = new BinaryLogger()
+            {
+                Parameters = $"LogFile={_logFile}",
+                CollectProjectImports = BinaryLogger.ProjectImportsCollectionMode.ZipFile,
+            };
+            var testProjectFmt = @"
+<Project>
+    <Target Name=""Build"" Inputs=""{0}"" Outputs=""testtaskoutputfile.txt"">
+        <ReadLinesFromFile
+            File=""{0}"" >
+            <Output
+                TaskParameter=""Lines""
+                ItemName=""ItemsFromFile""/>
+        </ReadLinesFromFile>
+        <WriteLinesToFile File=""testtaskoutputfile.txt"" Lines=""@(ItemsFromFile);abc;def;ghi""/>
+        <CreateItem Include=""testtaskoutputfile.txt"">
+            <Output TaskParameter=""Include"" ItemName=""EmbedInBinlog"" />
+        </CreateItem>
+        <CreateItem Include=""{0}"">
+            <Output TaskParameter=""Include"" ItemName=""EmbedInBinlog"" />
+        </CreateItem>
+        <CreateItem Include=""{1}"">
+            <Output TaskParameter=""Include"" ItemName=""EmbedInBinlog"" />
+        </CreateItem>
+    </Target>
+</Project>";
+            var testProject = string.Format(testProjectFmt, symlinkPath, symlinkLvl2Path);
+            ObjectModelHelpers.BuildProjectExpectSuccess(testProject, binaryLogger);
+            var projectImportsZipPath = Path.ChangeExtension(_logFile, ".ProjectImports.zip");
+            using var fileStream = new FileStream(projectImportsZipPath, FileMode.Open);
+            using var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read);
+
+            // Can't just compare `Name` because `ZipArchive` does not handle unix directory separators well
+            // thus producing garbled fully qualified paths in the actual .ProjectImports.zip entries
+            zipArchive.Entries.ShouldContain(zE => zE.Name.EndsWith("testtaskoutputfile.txt"));
+            zipArchive.Entries.ShouldContain(zE => zE.Name.EndsWith(symlinkName));
+            zipArchive.Entries.ShouldContain(zE => zE.Name.EndsWith(symlinkLvl2Name));
+        }
+
         [Fact]
         public void BinaryLoggerShouldNotThrowWhenMetadataCannotBeExpanded()
         {

src/Build/Logging/BinaryLogger/ProjectImportsCollector.cs

@@ -130,8 +130,7 @@ private void AddFileCore(string filePath)
                 return;
             }
 
-            var fileInfo = new FileInfo(filePath);
-            if (!fileInfo.Exists || fileInfo.Length == 0)
+            if (!NativeMethodsShared.ExistAndHasContent(filePath))
             {
                 _processedFiles.Add(filePath);
                 return;
@@ -145,11 +144,9 @@ private void AddFileCore(string filePath)
                 return;
             }
 
-            using (Stream entryStream = OpenArchiveEntry(filePath))
-            using (FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete))
-            {
-                content.CopyTo(entryStream);
-            }
+            using Stream entryStream = OpenArchiveEntry(filePath);
+            using FileStream content = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
+            content.CopyTo(entryStream);
         }
 
         /// <remarks>

src/Framework/NativeMethods.cs

@@ -10,6 +10,7 @@
 using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Runtime.Versioning;
+using System.Text;
 using System.Threading;
 
 using Microsoft.Build.Shared;
@@ -200,9 +201,16 @@ internal enum ProcessorArchitectures
         Unknown
     }
 
-#endregion
+    internal enum SymbolicLink
+    {
+        File = 0,
+        Directory = 1,
+        AllowUnprivilegedCreate = 2,
+    }
 
-#region Structs
+    #endregion
+
+    #region Structs
 
     /// <summary>
     /// Structure that contain information about the system on which we are running
@@ -1035,6 +1043,123 @@ internal static MemoryStatus GetMemoryStatus()
         return null;
     }
 
+    internal static bool ExistAndHasContent(string path)
+    {
+        var fileInfo = new FileInfo(path);
+
+        // File exist and has some content
+        return fileInfo.Exists &&
+               (fileInfo.Length > 0 ||
+                    // Or final destination of the link is nonempty file
+                    (
+                        IsSymLink(fileInfo) &&
+                        TryGetFinalLinkTarget(fileInfo, out string finalTarget, out _) &&
+                        File.Exists(finalTarget) &&
+                        new FileInfo(finalTarget).Length > 0
+                    )
+               );
+    }
+
+    internal static bool IsSymLink(FileInfo fileInfo)
+    {
+#if NET
+        return fileInfo.Exists && !string.IsNullOrEmpty(fileInfo.LinkTarget);
+#else
+        if (!IsWindows)
+        {
+            return false;
+        }
+
+        WIN32_FILE_ATTRIBUTE_DATA data = new WIN32_FILE_ATTRIBUTE_DATA();
+
+        return NativeMethods.GetFileAttributesEx(fileInfo.FullName, 0, ref data) &&
+               (data.fileAttributes & NativeMethods.FILE_ATTRIBUTE_DIRECTORY) == 0 &&
+               (data.fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == FILE_ATTRIBUTE_REPARSE_POINT;
+#endif
+    }
+
+    internal static bool IsSymLink(string path)
+    {
+        return IsSymLink(new FileInfo(path));
+    }
+
+    internal static bool TryGetFinalLinkTarget(FileInfo fileInfo, out string finalTarget, out string errorMessage)
+    {
+        if (!IsWindows)
+        {
+            errorMessage = null;
+#if NET
+            while(!string.IsNullOrEmpty(fileInfo.LinkTarget))
+            {
+                fileInfo = new FileInfo(fileInfo.LinkTarget);
+            }
+            finalTarget = fileInfo.FullName;
+            return true;
+#else
+
+            finalTarget = null;
+            return false;
+#endif
+        }
+
+        using SafeFileHandle handle = OpenFileThroughSymlinks(fileInfo.FullName);
+        if (handle.IsInvalid)
+        {
+            // Link is broken.
+            errorMessage = Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()).Message;
+            finalTarget = null;
+            return false;
+        }
+
+        const int initialBufferSize = 4096;
+        char[] targetPathBuffer = new char[initialBufferSize];
+        uint result = GetFinalPathNameByHandle(handle, targetPathBuffer);
+
+        // Buffer too small
+        if (result > targetPathBuffer.Length)
+        {
+            targetPathBuffer = new char[(int)result];
+            result = GetFinalPathNameByHandle(handle, targetPathBuffer);
+        }
+
+        // Error
+        if (result == 0)
+        {
+            errorMessage = Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()).Message;
+            finalTarget = null;
+            return false;
+        }
+
+        // Normalize \\?\ and \??\ syntax.
+        finalTarget = new string(targetPathBuffer, 0, (int)result).TrimStart(new char[] { '\\', '?' });
+        errorMessage = null;
+        return true;
+    }
+
+    internal static bool MakeSymbolicLink(string newFileName, string exitingFileName, ref string errorMessage)
+    {
+        bool symbolicLinkCreated;
+        if (IsWindows)
+        {
+            Version osVersion = Environment.OSVersion.Version;
+            SymbolicLink flags = SymbolicLink.File;
+            if (osVersion.Major >= 11 || (osVersion.Major == 10 && osVersion.Build >= 14972))
+            {
+                flags |= SymbolicLink.AllowUnprivilegedCreate;
+            }
+
+            symbolicLinkCreated = CreateSymbolicLink(newFileName, exitingFileName, flags);
+            errorMessage = symbolicLinkCreated ? null : Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()).Message;
+        }
+        else
+        {
+            symbolicLinkCreated = symlink(exitingFileName, newFileName) == 0;
+            errorMessage = symbolicLinkCreated ? null : "The link() library call failed with the following error code: " + Marshal.GetLastWin32Error();
+        }
+
+        return symbolicLinkCreated;
+    }
+
     /// <summary>
     /// Get the last write time of the fullpath to the file.
     /// </summary>
@@ -1111,6 +1236,23 @@ DateTime LastWriteFileUtcTime(string path)
         }
     }
 
+    /// <summary>
+    /// Get the SafeFileHandle for a file, while skipping reparse points (going directly to target file).
+    /// </summary>
+    /// <param name="fullPath">Full path to the file in the filesystem</param>
+    /// <returns>the SafeFileHandle for a file (target file in case of symlinks)</returns>
+    [SupportedOSPlatform("windows")]
+    private static SafeFileHandle OpenFileThroughSymlinks(string fullPath)
+    {
+        return CreateFile(fullPath,
+            GENERIC_READ,
+            FILE_SHARE_READ,
+            IntPtr.Zero,
+            OPEN_EXISTING,
+            FILE_ATTRIBUTE_NORMAL, /* No FILE_FLAG_OPEN_REPARSE_POINT; read through to content */
+            IntPtr.Zero);
+    }
+
     /// <summary>
     /// Get the last write time of the content pointed to by a file path.
     /// </summary>
@@ -1125,14 +1267,7 @@ private static DateTime GetContentLastWriteFileUtcTime(string fullPath)
     {
         DateTime fileModifiedTime = DateTime.MinValue;
 
-        using (SafeFileHandle handle =
-            CreateFile(fullPath,
-                GENERIC_READ,
-                FILE_SHARE_READ,
-                IntPtr.Zero,
-                OPEN_EXISTING,
-                FILE_ATTRIBUTE_NORMAL, /* No FILE_FLAG_OPEN_REPARSE_POINT; read through to content */
-                IntPtr.Zero))
+        using (SafeFileHandle handle = OpenFileThroughSymlinks(fullPath))
         {
             if (!handle.IsInvalid)
             {
@@ -1635,9 +1770,31 @@ out FILETIME lpLastWriteTime
     [SupportedOSPlatform("windows")]
     internal static extern bool SetThreadErrorMode(int newMode, out int oldMode);
 
-#endregion
+    [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
+    [return: MarshalAs(UnmanagedType.I1)]
+    [SupportedOSPlatform("windows")]
+    internal static extern bool CreateSymbolicLink(string symLinkFileName, string targetFileName, SymbolicLink dwFlags);
+
+    [DllImport("libc", SetLastError = true)]
+    internal static extern int symlink(string oldpath, string newpath);
+
+    internal const uint FILE_NAME_NORMALIZED = 0x0;
+
+    [SupportedOSPlatform("windows")]
+    static uint GetFinalPathNameByHandle(SafeFileHandle fileHandle, char[] filePath) =>
+        GetFinalPathNameByHandle(fileHandle, filePath, (uint) filePath.Length, FILE_NAME_NORMALIZED);
+
+    [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
+    [SupportedOSPlatform("windows")]
+    static extern uint GetFinalPathNameByHandle(
+        SafeFileHandle hFile,
+        [Out] char[] lpszFilePath,
+        uint cchFilePath,
+        uint dwFlags);
+
+    #endregion
 
-#region helper methods
+    #region helper methods
 
     internal static bool DirectoryExists(string fullPath)
     {

src/Tasks/Copy.cs

@@ -301,7 +301,7 @@ FileState destinationFileState  // The destination file
             // Create symbolic link if UseSymboliclinksIfPossible is true and hard link is not created
             if (!hardLinkCreated && UseSymboliclinksIfPossible)
             {
-                TryCopyViaLink(SymbolicLinkComment, MessageImportance.Normal, sourceFileState, destinationFileState, ref destinationFileExists, out symbolicLinkCreated, ref errorMessage, (source, destination, errMessage) => NativeMethods.MakeSymbolicLink(destination, source, ref errorMessage));
+                TryCopyViaLink(SymbolicLinkComment, MessageImportance.Normal, sourceFileState, destinationFileState, ref destinationFileExists, out symbolicLinkCreated, ref errorMessage, (source, destination, errMessage) => NativeMethodsShared.MakeSymbolicLink(destination, source, ref errorMessage));
                 if(!symbolicLinkCreated)
                 {
                     Log.LogMessage(MessageImportance.Normal, RetryingAsFileCopy, sourceFileState.Name, destinationFileState.Name, errorMessage);

src/Tasks/NativeMethods.cs

@@ -514,13 +514,6 @@ internal struct PROCESS_INFORMATION
         public int dwThreadId;
     }
 
-    internal enum SymbolicLink
-    {
-        File = 0,
-        Directory = 1,
-        AllowUnprivilegedCreate = 2,
-    }
-
     /// <summary>
     /// Interop methods.
     /// </summary>
@@ -819,40 +812,6 @@ internal static bool MakeHardLink(string newFileName, string exitingFileName, re
             return hardLinkCreated;
         }
 
-        //------------------------------------------------------------------------------
-        // CreateSymbolicLink
-        //------------------------------------------------------------------------------
-        [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
-        [return: MarshalAs(UnmanagedType.I1)]
-        internal static extern bool CreateSymbolicLink(string symLinkFileName, string targetFileName, SymbolicLink dwFlags);
-
-        [DllImport("libc", SetLastError = true)]
-        internal static extern int symlink(string oldpath, string newpath);
-
-        internal static bool MakeSymbolicLink(string newFileName, string exitingFileName, ref string errorMessage)
-        {
-            bool symbolicLinkCreated;
-            if (NativeMethodsShared.IsWindows)
-            {
-                Version osVersion = Environment.OSVersion.Version;
-                SymbolicLink flags = SymbolicLink.File;
-                if (osVersion.Major >= 11 || (osVersion.Major == 10 && osVersion.Build >= 14972))
-                {
-                    flags |= SymbolicLink.AllowUnprivilegedCreate;
-                }
-
-                symbolicLinkCreated = CreateSymbolicLink(newFileName, exitingFileName, flags);
-                errorMessage = symbolicLinkCreated ? null : Marshal.GetExceptionForHR(Marshal.GetHRForLastWin32Error()).Message;
-            }
-            else
-            {
-                symbolicLinkCreated = symlink(exitingFileName, newFileName) == 0;
-                errorMessage = symbolicLinkCreated ? null : "The link() library call failed with the following error code: " + Marshal.GetLastWin32Error();
-            }
-
-            return symbolicLinkCreated;
-        }
-
         //------------------------------------------------------------------------------
         // MoveFileEx
         //------------------------------------------------------------------------------