Add fallback keys to caching

src/Agent.Plugins/PipelineArtifact/PipelineArtifactPlugin.cs

@@ -5,12 +5,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.TeamFoundation.Build.WebApi;
-using Microsoft.VisualStudio.Services.BlobStore.Common;
-using Microsoft.VisualStudio.Services.Content.Common.Tracing;
-using Microsoft.VisualStudio.Services.BlobStore.WebApi;
 using Microsoft.VisualStudio.Services.Common;
-using Microsoft.VisualStudio.Services.WebApi;
-using Microsoft.TeamFoundation.DistributedTask.WebApi;
 using Microsoft.VisualStudio.Services.Agent.Util;
 using Agent.Sdk;
 using System.Text.RegularExpressions;

src/Agent.Plugins/PipelineCache/FingerprintCreator.cs

@@ -0,0 +1,234 @@
+using Agent.Sdk;
+using BuildXL.Cache.ContentStore.Interfaces.Utils;
+using Microsoft.VisualStudio.Services.BlobStore.Common;
+using Microsoft.VisualStudio.Services.PipelineCache.WebApi;
+using Minimatch;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Security.Cryptography;
+using System.Text;
+
+[assembly: InternalsVisibleTo("Test")]
+
+namespace Agent.Plugins.PipelineCache
+{
+    public static class FingerprintCreator
+    {
+        private static readonly bool isWindows = Helpers.IsWindowsPlatform(Environment.OSVersion);
+
+        // https://github.com/Microsoft/azure-pipelines-task-lib/blob/master/node/docs/findingfiles.md#matchoptions
+        private static readonly Options minimatchOptions = new Options
+        {
+            Dot = true,
+            NoBrace = true,
+            NoCase = isWindows,
+            AllowWindowsPaths = isWindows,
+        };
+
+        private static readonly char[] GlobChars = new [] { '*', '?', '[', ']' };
+
+        private static bool IsPathyChar(char c)
+        {
+            if (GlobChars.Contains(c)) return true;
+            if (c == Path.DirectorySeparatorChar) return true;
+            if (c == Path.AltDirectorySeparatorChar) return true;
+            if (c == Path.VolumeSeparatorChar) return true;
+            return !Path.GetInvalidFileNameChars().Contains(c);
+        }
+
+        internal static bool IsPathy(string keySegment)
+        {
+            if (keySegment.First() == '\'' && keySegment.Last() == '\'') return false;
+            if (keySegment.First() == '"' && keySegment.Last() == '"') return false;
+            if (keySegment.Any(c => !IsPathyChar(c))) return false;
+            //if (Uri.TryCreate(keySegment, UriKind.Absolute, out Uri dummy)) return false;
+            if (!keySegment.Contains(".")) return false;
+            if (keySegment.Last() == '.') return false;
+            return true;
+        }
+
+        internal static bool IsAbsolutePath(string path) =>
+               path.StartsWith("/", StringComparison.Ordinal)
+            || (isWindows && path.Length >= 3 && char.IsLetter(path[0]) && path[1] == ':' && path[2] == '\\');
+
+        internal static Func<string,bool> CreateMinimatchFilter(AgentTaskPluginExecutionContext context, string rule, bool invert)
+        {
+            Func<string,bool> filter = Minimatcher.CreateFilter(rule, minimatchOptions);
+            Func<string,bool> tracedFilter = (path) => {
+                bool result = invert ^ filter(path);
+                context.Verbose($"Path `{path}` is {(result ? "included" : "excluded")} because of pattern `{(invert ? "!" : "")}{rule}`.");
+                return result;
+            };
+
+            return tracedFilter;
+        }
+
+        internal static string MakePathAbsolute(string workingDirectory, string path)
+        {
+            if (workingDirectory != null)
+            {
+                path = $"{workingDirectory}{Path.DirectorySeparatorChar}{path}";
+            }
+
+            return path;
+        }
+
+        internal static Func<string,bool> CreateFilter(
+            AgentTaskPluginExecutionContext context,
+            string workingDirectory,
+            string includeRule,
+            IEnumerable<string> excludeRules)
+        {
+            Func<string,bool> includeFilter = CreateMinimatchFilter(context, includeRule, false);
+            Func<string,bool>[] excludeFilters = excludeRules.Select(excludeRule => 
+                CreateMinimatchFilter(context, excludeRule, true)).ToArray();
+            Func<string,bool> filter = (path) => includeFilter(path) && excludeFilters.All(f => f(path));
+            return filter;
+        }
+
+
+        internal static void DetermineEnumeration(
+            string workingDirectory,
+            string rootRule,
+            out string enumerateRootPath,
+            out string enumeratePattern,
+            out SearchOption enumerateDepth)
+        {
+            int firstGlob = rootRule.IndexOfAny(GlobChars);
+
+            // no globbing
+            if (firstGlob < 0)
+            {
+                if (workingDirectory == null)
+                {
+                    enumerateRootPath = Path.GetDirectoryName(rootRule);
+                }
+                else
+                {
+                    enumerateRootPath = workingDirectory;
+                }
+
+                enumeratePattern = Path.GetFileName(rootRule);
+                enumerateDepth = SearchOption.TopDirectoryOnly;
+            }
+            // starts with glob
+            else if(firstGlob == 0)
+            {
+                if(workingDirectory == null) throw new InvalidOperationException();
+                enumerateRootPath = workingDirectory;
+                enumeratePattern = "*";
+                enumerateDepth = SearchOption.AllDirectories;
+            }
+            else
+            {
+                int rootDirLength = rootRule.Substring(0,firstGlob).LastIndexOfAny( new [] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar});
+                enumerateRootPath = rootRule.Substring(0,rootDirLength);
+                enumeratePattern = "*";
+                enumerateDepth = SearchOption.AllDirectories;
+            }
+        }
+
+        public static Fingerprint CreateFingerprint(
+            AgentTaskPluginExecutionContext context,
+            IEnumerable<string> keySegments)
+        {
+            var sha256 = new SHA256Managed();
+
+            string workingDirectoryValue = context.Variables.GetValueOrDefault(
+                "system.defaultworkingdirectory" // Constants.Variables.System.DefaultWorkingDirectory
+                )?.Value;
+
+            var resolvedSegments = new List<string>();
+
+            foreach (string keySegment in keySegments)
+            {
+                bool isPathy = IsPathy(keySegment);
+                bool isWildCard = keySegment.Equals("**", StringComparison.Ordinal);
+
+                if (isWildCard)
+                {
+                    resolvedSegments.Add("**");
+                }
+                else if (isPathy)
+                {
+                    context.Verbose($"Interpretting `{keySegment}` as a path.");
+
+                    var segment = new StringBuilder();
+                    bool foundFile = false;
+
+                    string[] pathSections = keySegment.Split(new []{';'}, StringSplitOptions.RemoveEmptyEntries);
+                    foreach(string pathSection in pathSections)
+                    {
+                        string[] pathRules = pathSection.Split(new []{','}, StringSplitOptions.RemoveEmptyEntries);
+                        string rootRule = pathRules.First();
+                        if(rootRule.Length == 0 || rootRule[1] == '!')
+                        {
+                            throw new ArgumentException();
+                        }
+
+                        string workingDirectory = null;
+                        if (!IsAbsolutePath(rootRule))
+                        {
+                            workingDirectory = workingDirectoryValue;
+                        }
+
+                        string absoluteRootRule = MakePathAbsolute(workingDirectory, rootRule);
+                        context.Verbose($"Expanded include rule is `{absoluteRootRule}`.");
+                        IEnumerable<string> absoluteExcludeRules = pathRules.Skip(1).Select(r => MakePathAbsolute(workingDirectory, r));
+                        Func<string,bool> filter = CreateFilter(context, workingDirectory, absoluteRootRule, absoluteExcludeRules);
+
+                        DetermineEnumeration(
+                            workingDirectory,
+                            absoluteRootRule,
+                            out string enumerateRootPath,
+                            out string enumeratePattern,
+                            out SearchOption enumerateDepth);
+
+                        context.Verbose($"Enumerating starting at root `{enumerateRootPath}` with pattern `{enumeratePattern}`.");
+                        IEnumerable<string> files = Directory.EnumerateFiles(enumerateRootPath, enumeratePattern, enumerateDepth);
+                        files = files.Where(f => filter(f)).Distinct();
+
+                        foreach(string path in files)
+                        {
+                            foundFile = true;
+
+                            using (var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read))
+                            {
+                                byte[] hash = sha256.ComputeHash(fs);
+                                string displayPath = workingDirectory == null ? path : path.Substring(enumerateRootPath.Length + 1);
+                                segment.Append($"\nSHA256({displayPath})=[{fs.Length}]{hash.ToHex()}");
+                            }
+                        }
+                    }
+
+                    if (!foundFile)
+                    {
+                        throw new FileNotFoundException("No files found.");
+                    }
+
+                    string fileHashString = segment.ToString();
+                    string fileHashStringHash = SummarizeString(fileHashString);
+                    context.Output($"File hashes summarized as `{fileHashStringHash}` from BASE64(SHA256(`{fileHashString}`))");
+                    resolvedSegments.Add(fileHashStringHash);
+                } 
+                else
+                {
+                    context.Verbose($"Interpretting `{keySegment}` as a string.");
+                    resolvedSegments.Add($"{keySegment}");
+                }
+            }
+
+            return new Fingerprint() { Segments = resolvedSegments.ToArray() };
+        }
+
+        internal static string SummarizeString(string input)
+        {
+            var sha256 = new SHA256Managed();
+            byte[] fileHashStringBytes = sha256.ComputeHash(Encoding.UTF8.GetBytes(input));
+            return Convert.ToBase64String(fileHashStringBytes);
+        }
+    }
+}

src/Agent.Plugins/PipelineCache/PipelineCacheServer.cs

@@ -1,35 +1,25 @@
-using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using System.Threading;
-using System;
 using Agent.Plugins.PipelineArtifact;
-using Agent.Plugins.PipelineArtifact.Telemetry;
 using Agent.Plugins.PipelineCache.Telemetry;
 using Agent.Sdk;
-using Microsoft.TeamFoundation.Build.WebApi;
-using Microsoft.TeamFoundation.DistributedTask.WebApi;
-using Microsoft.VisualStudio.Services.Agent.Util;
 using Microsoft.VisualStudio.Services.BlobStore.Common;
 using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;
 using Microsoft.VisualStudio.Services.BlobStore.WebApi;
 using Microsoft.VisualStudio.Services.Content.Common;
-using Microsoft.VisualStudio.Services.Common;
 using Microsoft.VisualStudio.Services.Content.Common.Tracing;
 using Microsoft.VisualStudio.Services.PipelineCache.WebApi;
 using Microsoft.VisualStudio.Services.WebApi;
-using Newtonsoft.Json;
 
 namespace Agent.Plugins.PipelineCache
 {
     public class PipelineCacheServer
     {
         internal async Task UploadAsync(
             AgentTaskPluginExecutionContext context,
-            IEnumerable<string> key,
+            Fingerprint fingerprint,
             string path,
-            string salt,
             CancellationToken cancellationToken)
         {
             VssConnection connection = context.VssConnection;
@@ -40,20 +30,15 @@ internal async Task UploadAsync(
             using (clientTelemetry)
             {
                 // Check if the key exists.
-                GetPipelineCacheArtifactOptions getOptions = new GetPipelineCacheArtifactOptions
-                {
-                    Key = key,
-                    Salt = salt,
-                };
                 PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord<PipelineCacheActionRecord>((level, uri, type) =>
                         new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
-                PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(getOptions, cancellationToken, cacheRecordGet);
+                PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] {fingerprint}, cancellationToken, cacheRecordGet);
                 // Send results to CustomerIntelligence
                 context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
                 //If cache exists, return.
                 if (getResult != null)
                 {
-                    context.Output($"Cache with fingerprint {getResult.Fingerprint} already exists.");
+                    context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
                     return;
                 }
 
@@ -69,11 +54,10 @@ internal async Task UploadAsync(
 
                 CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions
                 {
-                    Key = key,
+                    Fingerprint = fingerprint,
                     RootId = result.RootId,
                     ManifestId = result.ManifestId,
                     ProofNodes = result.ProofNodes.ToArray(),
-                    Salt = salt
                 };
 
                 // Cache the artifact
@@ -90,34 +74,29 @@ internal async Task UploadAsync(
 
         internal async Task DownloadAsync(
             AgentTaskPluginExecutionContext context,
-            IEnumerable<string> key,
+            Fingerprint[] fingerprints,
             string path,
-            string salt,
             string cacheHitVariable,
             CancellationToken cancellationToken)
         {
             VssConnection connection = context.VssConnection;
             BlobStoreClientTelemetry clientTelemetry;
             DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry);
             PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
-            GetPipelineCacheArtifactOptions options = new GetPipelineCacheArtifactOptions
-            {
-                Key = key,
-                Salt = salt,
-            };
-
+
             using (clientTelemetry)
             {
                 PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord<PipelineCacheActionRecord>((level, uri, type) =>
                         new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
-                PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
+                PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord);
 
                 // Send results to CustomerIntelligence
                 context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
 
                 if (result != null)
                 {
-                    context.Output($"Manifest ID is: {result.ManifestId.ValueString}");
+                    context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`");
+                    context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}");
                     PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineCacheActionRecord>((level, uri, type) =>
                         new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
                     await clientTelemetry.MeasureActionAsync(
@@ -135,7 +114,25 @@ await clientTelemetry.MeasureActionAsync(
 
                 if (!string.IsNullOrEmpty(cacheHitVariable))
                 {
-                    context.SetVariable(cacheHitVariable, result != null ? "true" : "false");
+                    if (result == null) {
+                        context.SetVariable(cacheHitVariable, "false");
+                    }  else  {
+                        context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`");
+
+                        bool foundExact = false;
+                        foreach(var fingerprint in fingerprints)
+                        {
+                            context.Verbose($"This fingerprint: `{fingerprint.ToString()}`");
+
+                            if(fingerprint == result.Fingerprint)
+                            {
+                                foundExact = true;
+                                break;
+                            }
+                        }
+
+                        context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact");
+                    }
                 }
             }
         }