diff --git a/nuget/Dockerfile b/nuget/Dockerfile index 85ca8c66db..2eb63dc776 100644 --- a/nuget/Dockerfile +++ b/nuget/Dockerfile @@ -4,34 +4,39 @@ USER root ENV DEPENDABOT_NATIVE_HELPERS_PATH="/opt" -# Install .NET SDK dependencies +# install dependencies +RUN source /etc/os-release \ + && curl --location --output /tmp/packages-microsoft-prod.deb "https://packages.microsoft.com/config/ubuntu/$VERSION_ID/packages-microsoft-prod.deb" \ + && dpkg -i /tmp/packages-microsoft-prod.deb \ + && rm /tmp/packages-microsoft-prod.deb RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - libicu-dev=70.1-2 \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends \ + jq \ + libicu-dev=70.1-2 \ + powershell \ + && rm -rf /var/lib/apt/lists/* # Install .NET SDK ARG DOTNET_SDK_VERSION=8.0.303 ARG DOTNET_SDK_INSTALL_URL=https://dot.net/v1/dotnet-install.sh ENV DOTNET_INSTALL_DIR=/usr/local/dotnet/current +ENV DOTNET_INSTALL_SCRIPT_PATH=/tmp/dotnet-install.sh ENV DOTNET_NOLOGO=true ENV DOTNET_ROOT="${DOTNET_INSTALL_DIR}" ENV DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true ENV DOTNET_CLI_TELEMETRY_OPTOUT=true ENV NUGET_SCRATCH=/opt/nuget/helpers/tmp -RUN cd /tmp \ - && curl --location --output dotnet-install.sh "${DOTNET_SDK_INSTALL_URL}" \ - && chmod +x dotnet-install.sh \ - && mkdir -p "${DOTNET_INSTALL_DIR}" \ - && ./dotnet-install.sh --version "${DOTNET_SDK_VERSION}" --install-dir "${DOTNET_INSTALL_DIR}" \ - && rm dotnet-install.sh \ - && chown -R dependabot:dependabot "${DOTNET_INSTALL_DIR}/sdk" - +RUN curl --location --output "${DOTNET_INSTALL_SCRIPT_PATH}" "${DOTNET_SDK_INSTALL_URL}" \ + && chmod +x "${DOTNET_INSTALL_SCRIPT_PATH}" \ + && mkdir -p "${DOTNET_INSTALL_DIR}" \ + && "${DOTNET_INSTALL_SCRIPT_PATH}" --version "${DOTNET_SDK_VERSION}" --install-dir "${DOTNET_INSTALL_DIR}" \ + && chown -R dependabot:dependabot "$DOTNET_INSTALL_DIR" ENV PATH="${PATH}:${DOTNET_INSTALL_DIR}" RUN dotnet --list-runtimes RUN dotnet --list-sdks +# build tools USER dependabot COPY --chown=dependabot:dependabot nuget/helpers /opt/nuget/helpers RUN bash /opt/nuget/helpers/build @@ -39,3 +44,9 @@ RUN bash /opt/nuget/helpers/build COPY --chown=dependabot:dependabot nuget $DEPENDABOT_HOME/nuget COPY --chown=dependabot:dependabot common $DEPENDABOT_HOME/common COPY --chown=dependabot:dependabot updater $DEPENDABOT_HOME/dependabot-updater + +# redirect entrypoint +RUN mv bin/run bin/run-original +COPY --chown=dependabot:dependabot nuget/script/* $DEPENDABOT_HOME/dependabot-updater/bin/ +COPY --chown=dependabot:dependabot nuget/updater/* $DEPENDABOT_HOME/dependabot-updater/bin/ +RUN chmod +x $DEPENDABOT_HOME/dependabot-updater/bin/run diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Run.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Run.cs new file mode 100644 index 0000000000..a84e312993 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Run.cs @@ -0,0 +1,132 @@ +using System.Text; +using System.Text.Json; + +using NuGetUpdater.Core.Run; +using NuGetUpdater.Core.Run.ApiModel; +using NuGetUpdater.Core.Test; +using NuGetUpdater.Core.Test.Update; + +using Xunit; + +namespace NuGetUpdater.Cli.Test; + +using TestFile = (string Path, string Content); + +public partial class EntryPointTests +{ + public class Run + { + [Fact] + public async Task Run_Simple() + { + // verify we can pass command line arguments and hit the appropriate URLs + await RunAsync( + packages: + [ + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0"), + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.1", "net8.0"), + ], + files: + [ + ("src/project.csproj", """ + + + net8.0 + + + + + + """) + ], + job: new Job() + { + PackageManager = "nuget", + AllowedUpdates = [ + new() + { + UpdateType = "all" + } + ], + Source = new() + { + Provider = "github", + Repo = "test", + Directory = "src", + } + }, + expectedUrls: + [ + "/update_jobs/TEST-ID/update_dependency_list", + "/update_jobs/TEST-ID/increment_metric", + "/update_jobs/TEST-ID/create_pull_request", + "/update_jobs/TEST-ID/mark_as_processed", + ] + ); + } + + private static async Task RunAsync(TestFile[] files, Job job, string[] expectedUrls, MockNuGetPackage[]? packages = null) + { + using var tempDirectory = new TemporaryDirectory(); + + // write test files + foreach (var testFile in files) + { + var fullPath = Path.Join(tempDirectory.DirectoryPath, testFile.Path); + var directory = Path.GetDirectoryName(fullPath)!; + Directory.CreateDirectory(directory); + await File.WriteAllTextAsync(fullPath, testFile.Content); + } + + // write job file + var jobPath = Path.Combine(tempDirectory.DirectoryPath, "job.json"); + await File.WriteAllTextAsync(jobPath, JsonSerializer.Serialize(new { Job = job }, RunWorker.SerializerOptions)); + + // save packages + await UpdateWorkerTestBase.MockNuGetPackagesInDirectory(packages, tempDirectory.DirectoryPath); + + var actualUrls = new List(); + using var http = TestHttpServer.CreateTestStringServer(url => + { + actualUrls.Add(new Uri(url).PathAndQuery); + return (200, "ok"); + }); + var args = new List() + { + "run", + "--job-path", + jobPath, + "--repo-contents-path", + tempDirectory.DirectoryPath, + "--api-url", + http.BaseUrl, + "--job-id", + "TEST-ID", + "--output-path", + Path.Combine(tempDirectory.DirectoryPath, "output.json"), + "--base-commit-sha", + "BASE-COMMIT-SHA", + "--verbose" + }; + + var output = new StringBuilder(); + // redirect stdout + var originalOut = Console.Out; + Console.SetOut(new StringWriter(output)); + int result = -1; + try + { + result = await Program.Main(args.ToArray()); + } + catch + { + // restore stdout + Console.SetOut(originalOut); + throw; + } + + Assert.True(result == 0, output.ToString()); + Assert.Equal(expectedUrls, actualUrls); + } + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs new file mode 100644 index 0000000000..c15d6b764f --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs @@ -0,0 +1,42 @@ +using System.CommandLine; + +using NuGetUpdater.Core; +using NuGetUpdater.Core.Run; + +namespace NuGetUpdater.Cli.Commands; + +internal static class RunCommand +{ + internal static readonly Option JobPathOption = new("--job-path") { IsRequired = true }; + internal static readonly Option RepoContentsPathOption = new("--repo-contents-path") { IsRequired = true }; + internal static readonly Option ApiUrlOption = new("--api-url") { IsRequired = true }; + internal static readonly Option JobIdOption = new("--job-id") { IsRequired = true }; + internal static readonly Option OutputPathOption = new("--output-path") { IsRequired = true }; + internal static readonly Option BaseCommitShaOption = new("--base-commit-sha") { IsRequired = true }; + internal static readonly Option VerboseOption = new("--verbose", getDefaultValue: () => false); + + internal static Command GetCommand(Action setExitCode) + { + Command command = new("run", "Runs a full dependabot job.") + { + JobPathOption, + RepoContentsPathOption, + ApiUrlOption, + JobIdOption, + OutputPathOption, + BaseCommitShaOption, + VerboseOption + }; + + command.TreatUnmatchedTokensAsErrors = true; + + command.SetHandler(async (jobPath, repoContentsPath, apiUrl, jobId, outputPath, baseCommitSha, verbose) => + { + var apiHandler = new HttpApiHandler(apiUrl.ToString(), jobId); + var worker = new RunWorker(apiHandler, new Logger(verbose)); + await worker.RunAsync(jobPath, repoContentsPath, baseCommitSha, outputPath); + }, JobPathOption, RepoContentsPathOption, ApiUrlOption, JobIdOption, OutputPathOption, BaseCommitShaOption, VerboseOption); + + return command; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs index ff0235509b..62561336f2 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Program.cs @@ -17,6 +17,7 @@ internal static async Task Main(string[] args) DiscoverCommand.GetCommand(setExitCode), AnalyzeCommand.GetCommand(setExitCode), UpdateCommand.GetCommand(setExitCode), + RunCommand.GetCommand(setExitCode), }; command.TreatUnmatchedTokensAsErrors = true; diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs new file mode 100644 index 0000000000..bf75154bcd --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs @@ -0,0 +1,223 @@ +using System.Text; +using System.Text.Json; +using System.Xml.Linq; + +using NuGetUpdater.Core.Run; +using NuGetUpdater.Core.Run.ApiModel; +using NuGetUpdater.Core.Test.Update; + +using Xunit; + +namespace NuGetUpdater.Core.Test.Run; + +using TestFile = (string Path, string Content); + +public class RunWorkerTests +{ + [Fact] + public async Task UpdateSinglePackageProducedExpectedAPIMessages() + { + var repoMetadata = XElement.Parse(""""""); + await RunAsync( + packages: + [ + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0", additionalMetadata: [repoMetadata]), + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.1", "net8.0", additionalMetadata: [repoMetadata]), + ], + job: new Job() + { + PackageManager = "nuget", + Source = new() + { + Provider = "github", + Repo = "test/repo", + Directory = "some-dir", + }, + AllowedUpdates = + [ + new() { UpdateType = "all" } + ] + }, + files: + [ + ("some-dir/project.csproj", """ + + + net8.0 + + + + + + """) + ], + expectedResult: new RunResult() + { + Base64DependencyFiles = + [ + new DependencyFile() + { + Directory = "/some-dir", + Name = "project.csproj", + Content = Convert.ToBase64String(Encoding.UTF8.GetBytes(""" + + + net8.0 + + + + + + """)) + } + ], + BaseCommitSha = "TEST-COMMIT-SHA", + }, + expectedApiMessages: + [ + new UpdatedDependencyList() + { + Dependencies = + [ + new ReportedDependency() + { + Name = "Some.Package", + Version = "1.0.0", + Requirements = + [ + new ReportedRequirement() + { + Requirement = "1.0.0", + File = "/some-dir/project.csproj", + Groups = ["dependencies"], + } + ] + } + ], + DependencyFiles = ["/some-dir/project.csproj"], + }, + new IncrementMetric() + { + Metric = "updater.started", + Tags = new() + { + ["operation"] = "group_update_all_versions" + } + }, + new CreatePullRequest() + { + Dependencies = + [ + new ReportedDependency() + { + Name = "Some.Package", + Version = "1.0.1", + Requirements = + [ + new ReportedRequirement() + { + Requirement = "1.0.1", + File = "/some-dir/project.csproj", + Groups = ["dependencies"], + Source = new() + { + SourceUrl = "https://nuget.example.com/some-package", + Type = "nuget_repo", + } + } + ], + PreviousVersion = "1.0.0", + PreviousRequirements = + [ + new ReportedRequirement() + { + Requirement = "1.0.0", + File = "/some-dir/project.csproj", + Groups = ["dependencies"], + } + ], + } + ], + UpdatedDependencyFiles = + [ + new DependencyFile() + { + Name = "project.csproj", + Directory = "some-dir", + Content = """ + + + net8.0 + + + + + + """, + }, + ], + BaseCommitSha = "TEST-COMMIT-SHA", + CommitMessage = "TODO: message", + PrTitle = "TODO: title", + PrBody = "TODO: body", + }, + new MarkAsProcessed() + { + BaseCommitSha = "TEST-COMMIT-SHA", + } + ] + ); + } + + private static async Task RunAsync(Job job, TestFile[] files, RunResult expectedResult, object[] expectedApiMessages, MockNuGetPackage[]? packages = null) + { + // arrange + using var tempDirectory = new TemporaryDirectory(); + await UpdateWorkerTestBase.MockNuGetPackagesInDirectory(packages, tempDirectory.DirectoryPath); + foreach (var (path, content) in files) + { + var fullPath = Path.Combine(tempDirectory.DirectoryPath, path); + var directory = Path.GetDirectoryName(fullPath)!; + Directory.CreateDirectory(directory); + await File.WriteAllTextAsync(fullPath, content); + } + + // act + var testApiHandler = new TestApiHandler(); + var worker = new RunWorker(testApiHandler, new Logger(verbose: false)); + var repoContentsPath = new DirectoryInfo(tempDirectory.DirectoryPath); + var actualResult = await worker.RunAsync(job, repoContentsPath, "TEST-COMMIT-SHA"); + var actualApiMessages = testApiHandler.ReceivedMessages.ToArray(); + + // assert + var actualRunResultJson = JsonSerializer.Serialize(actualResult); + var expectedRunResultJson = JsonSerializer.Serialize(expectedResult); + Assert.Equal(expectedRunResultJson, actualRunResultJson); + for (int i = 0; i < Math.Min(actualApiMessages.Length, expectedApiMessages.Length); i++) + { + var actualMessage = actualApiMessages[i]; + var expectedMessage = expectedApiMessages[i]; + Assert.Equal(expectedMessage.GetType(), actualMessage.Type); + + var expectedContent = SerializeObjectAndType(expectedMessage); + var actualContent = SerializeObjectAndType(actualMessage.Object); + Assert.Equal(expectedContent, actualContent); + } + + if (actualApiMessages.Length > expectedApiMessages.Length) + { + var extraApiMessages = actualApiMessages.Skip(expectedApiMessages.Length).Select(m => SerializeObjectAndType(m.Object)).ToArray(); + Assert.Fail($"Expected {expectedApiMessages.Length} API messages, but got {extraApiMessages.Length} extra:\n\t{string.Join("\n\t", extraApiMessages)}"); + } + if (expectedApiMessages.Length > actualApiMessages.Length) + { + var missingApiMessages = expectedApiMessages.Skip(actualApiMessages.Length).Select(m => SerializeObjectAndType(m)).ToArray(); + Assert.Fail($"Expected {expectedApiMessages.Length} API messages, but only got {actualApiMessages.Length}; missing:\n\t{string.Join("\n\t", missingApiMessages)}"); + } + } + + private static string SerializeObjectAndType(object obj) + { + return $"{obj.GetType().Name}:{JsonSerializer.Serialize(obj)}"; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs new file mode 100644 index 0000000000..048912ec0d --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs @@ -0,0 +1,60 @@ +using NuGetUpdater.Core.Run; + +using Xunit; + +namespace NuGetUpdater.Core.Test.Run; + +public class SerializationTests +{ + [Fact] + public void DeserializeJob() + { + var jobWrapper = RunWorker.Deserialize(""" + { + "job": { + "package-manager": "nuget", + "allowed-updates": [ + { + "update-type": "all" + } + ], + "debug": false, + "dependency-groups": [], + "dependencies": null, + "dependency-group-to-refresh": null, + "existing-pull-requests": [], + "existing-group-pull-requests": [], + "experiments": null, + "ignore-conditions": [], + "lockfile-only": false, + "requirements-update-strategy": null, + "security-advisories": [], + "security-updates-only": false, + "source": { + "provider": "github", + "repo": "some-org/some-repo", + "directory": "specific-sdk", + "hostname": null, + "api-endpoint": null + }, + "update-subdependencies": false, + "updating-a-pull-request": false, + "vendor-dependencies": false, + "reject-external-code": false, + "repo-private": false, + "commit-message-options": null, + "credentials-metadata": [ + { + "host": "github.com", + "type": "git_source" + } + ], + "max-updater-run-time": 0 + } + } + """); + Assert.Equal("github", jobWrapper.Job.Source.Provider); + Assert.Equal("some-org/some-repo", jobWrapper.Job.Source.Repo); + Assert.Equal("specific-sdk", jobWrapper.Job.Source.Directory); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/TestApiHandler.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/TestApiHandler.cs new file mode 100644 index 0000000000..52954bf637 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/TestApiHandler.cs @@ -0,0 +1,35 @@ +using NuGetUpdater.Core.Run; +using NuGetUpdater.Core.Run.ApiModel; + +namespace NuGetUpdater.Core.Test; + +internal class TestApiHandler : IApiHandler +{ + private readonly List<(Type, object)> _receivedMessages = new(); + + public IEnumerable<(Type Type, object Object)> ReceivedMessages => _receivedMessages; + + public Task UpdateDependencyList(UpdatedDependencyList updatedDependencyList) + { + _receivedMessages.Add((typeof(UpdatedDependencyList), updatedDependencyList)); + return Task.CompletedTask; + } + + public Task IncrementMetric(IncrementMetric incrementMetric) + { + _receivedMessages.Add((typeof(IncrementMetric), incrementMetric)); + return Task.CompletedTask; + } + + public Task CreatePullRequest(CreatePullRequest createPullRequest) + { + _receivedMessages.Add((typeof(CreatePullRequest), createPullRequest)); + return Task.CompletedTask; + } + + public Task MarkAsProcessed(MarkAsProcessed markAsProcessed) + { + _receivedMessages.Add((typeof(MarkAsProcessed), markAsProcessed)); + return Task.CompletedTask; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/UpdatedDependencyListTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/UpdatedDependencyListTests.cs new file mode 100644 index 0000000000..dd27443ac8 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/UpdatedDependencyListTests.cs @@ -0,0 +1,69 @@ +using System.Text.Json; + +using NuGetUpdater.Core.Discover; +using NuGetUpdater.Core.Run; +using NuGetUpdater.Core.Run.ApiModel; + +using Xunit; + +namespace NuGetUpdater.Core.Test.Run; + +public class UpdatedDependencyListTests +{ + [Fact] + public void GetUpdatedDependencyListFromDiscovery() + { + var discovery = new WorkspaceDiscoveryResult() + { + Path = "src", + IsSuccess = true, + Projects = [ + new() + { + FilePath = "project.csproj", + Dependencies = [ + new("Microsoft.Extensions.DependencyModel", "6.0.0", DependencyType.PackageReference, TargetFrameworks: ["net6.0"]), + new("System.Text.Json", "6.0.0", DependencyType.Unknown, TargetFrameworks: ["net6.0"], IsTransitive: true), + ], + IsSuccess = true, + Properties = [], + TargetFrameworks = ["net8.0"], + ReferencedProjectPaths = [], + } + ] + }; + var updatedDependencyList = RunWorker.GetUpdatedDependencyListFromDiscovery(discovery); + var expectedDependencyList = new UpdatedDependencyList() + { + Dependencies = + [ + new ReportedDependency() + { + Name = "Microsoft.Extensions.DependencyModel", + Version = "6.0.0", + Requirements = + [ + new ReportedRequirement() + { + Requirement = "6.0.0", + File = "/src/project.csproj", + Groups = ["dependencies"], + } + ] + }, + new ReportedDependency() + { + Name = "System.Text.Json", + Version = "6.0.0", + Requirements = [], + } + ], + DependencyFiles = ["/src/project.csproj"], + }; + + // doing JSON comparison makes this easier; we don't have to define custom record equality and we get an easy diff + var actualJson = JsonSerializer.Serialize(updatedDependencyList); + var expectedJson = JsonSerializer.Serialize(expectedDependencyList); + Assert.Equal(expectedJson, actualJson); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/PathHelperTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/PathHelperTests.cs new file mode 100644 index 0000000000..dfcec6fb8c --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/PathHelperTests.cs @@ -0,0 +1,22 @@ +using Xunit; + +namespace NuGetUpdater.Core.Test.Utilities; + +public class PathHelperTests +{ + [Theory] + [InlineData("a/b/c", "a/b/c")] + [InlineData("a/b/../c", "a/c")] + [InlineData("a/..//c", "c")] + [InlineData("/a/b/c", "/a/b/c")] + [InlineData("/a/b/../c", "/a/c")] + [InlineData("/a/..//c", "/c")] + [InlineData("a/b/./c", "a/b/c")] + [InlineData("a/../../b", "b")] + [InlineData("../../../a/b", "a/b")] + public void VerifyNormalizeUnixPathParts(string input, string expected) + { + var actual = input.NormalizeUnixPathParts(); + Assert.Equal(expected, actual); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs index 0a45ce7cfd..d959ce52e6 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs @@ -33,6 +33,12 @@ public async Task RunAsync(string repoRoot, string discoveryPath, string depende { var discovery = await DeserializeJsonFileAsync(discoveryPath, nameof(WorkspaceDiscoveryResult)); var dependencyInfo = await DeserializeJsonFileAsync(dependencyPath, nameof(DependencyInfo)); + var analysisResult = await RunAsync(repoRoot, discovery, dependencyInfo); + await WriteResultsAsync(analysisDirectory, dependencyInfo.Name, analysisResult, _logger); + } + + public async Task RunAsync(string repoRoot, WorkspaceDiscoveryResult discovery, DependencyInfo dependencyInfo) + { var startingDirectory = PathHelper.JoinPath(repoRoot, discovery.Path); _logger.Log($"Starting analysis of {dependencyInfo.Name}..."); @@ -61,7 +67,7 @@ public async Task RunAsync(string repoRoot, string discoveryPath, string depende bool isProjectUpdateNecessary = IsUpdateNecessary(dependencyInfo, projectsWithDependency); var isUpdateNecessary = isProjectUpdateNecessary || dotnetToolsHasDependency || globalJsonHasDependency; using var nugetContext = new NuGetContext(startingDirectory); - AnalysisResult result; + AnalysisResult analysisResult; try { if (isUpdateNecessary) @@ -134,7 +140,7 @@ public async Task RunAsync(string repoRoot, string discoveryPath, string depende // should track the dependenciesToUpdate as they have already been analyzed. } - result = new AnalysisResult + analysisResult = new AnalysisResult { UpdatedVersion = updatedVersion?.ToNormalizedString() ?? dependencyInfo.Version, CanUpdate = updatedVersion is not null, @@ -146,7 +152,7 @@ public async Task RunAsync(string repoRoot, string discoveryPath, string depende when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) { // TODO: consolidate this error handling between AnalyzeWorker, DiscoveryWorker, and UpdateWorker - result = new AnalysisResult + analysisResult = new AnalysisResult { ErrorType = ErrorType.AuthenticationFailure, ErrorDetails = "(" + string.Join("|", nugetContext.PackageSources.Select(s => s.Source)) + ")", @@ -156,9 +162,8 @@ public async Task RunAsync(string repoRoot, string discoveryPath, string depende }; } - await WriteResultsAsync(analysisDirectory, dependencyInfo.Name, result, _logger); - _logger.Log($"Analysis complete."); + return analysisResult; } private static bool IsUpdateNecessary(DependencyInfo dependencyInfo, ImmutableArray projectsWithDependency) @@ -393,7 +398,7 @@ internal static async Task> FindUpdatedDependenciesAs // Create distinct list of dependencies taking the highest version of each var dependencyResult = await DependencyFinder.GetDependenciesAsync( - workspacePath, + repoRoot, projectPath, projectFrameworks, packageIds, diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs index 87fb53778f..c320c69d55 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs @@ -8,7 +8,7 @@ namespace NuGetUpdater.Core.Analyze; internal static class DependencyFinder { public static async Task>> GetDependenciesAsync( - string workspacePath, + string repoRoot, string projectPath, IEnumerable frameworks, ImmutableHashSet packageIds, @@ -26,7 +26,7 @@ public static async Task RunAsync(string repoRootPath, string workspacePath) { MSBuildHelper.RegisterMSBuild(Environment.CurrentDirectory, repoRootPath); @@ -102,11 +102,16 @@ public async Task RunAsync(string repoRootPath, string workspacePath, string out }; } - await WriteResultsAsync(repoRootPath, outputPath, result); - _logger.Log("Discovery complete."); - _processedProjectPaths.Clear(); + + return result; + } + + public async Task RunAsync(string repoRootPath, string workspacePath, string outputPath) + { + var result = await RunAsync(repoRootPath, workspacePath); + await WriteResultsAsync(repoRootPath, outputPath, result); } /// diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj index faa7e17bb5..c91a5e6b05 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj @@ -21,7 +21,8 @@ - + + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/AllowedUpdate.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/AllowedUpdate.cs new file mode 100644 index 0000000000..5ae9ab2b09 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/AllowedUpdate.cs @@ -0,0 +1,6 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record AllowedUpdate +{ + public string UpdateType { get; init; } = "all"; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/CreatePullRequest.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/CreatePullRequest.cs new file mode 100644 index 0000000000..905b4fccff --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/CreatePullRequest.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record CreatePullRequest +{ + public required ReportedDependency[] Dependencies { get; init; } + [JsonPropertyName("updated-dependency-files")] + public required DependencyFile[] UpdatedDependencyFiles { get; init; } + [JsonPropertyName("base-commit-sha")] + public required string BaseCommitSha { get; init; } + [JsonPropertyName("commit-message")] + public required string CommitMessage { get; init; } + [JsonPropertyName("pr-title")] + public required string PrTitle { get; init; } + [JsonPropertyName("pr-body")] + public required string PrBody { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFile.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFile.cs new file mode 100644 index 0000000000..1dd84c19a1 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFile.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record DependencyFile +{ + public required string Name { get; init; } + public required string Content { get; init; } + public required string Directory { get; init; } + public string Type { get; init; } = "file"; // TODO: enum + [JsonPropertyName("support_file")] + public bool SupportFile { get; init; } = false; + [JsonPropertyName("content_encoding")] + public string ContentEncoding { get; init; } = "utf-8"; + public bool Deleted { get; init; } = false; + public string Operation { get; init; } = "update"; // TODO: enum + public string? Mode { get; init; } = null; // TODO: what is this? +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/IncrementMetric.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/IncrementMetric.cs new file mode 100644 index 0000000000..b7298a478e --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/IncrementMetric.cs @@ -0,0 +1,7 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record IncrementMetric +{ + public required string Metric { get; init; } + public Dictionary Tags { get; init; } = new(); +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Job.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Job.cs new file mode 100644 index 0000000000..b612717df8 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Job.cs @@ -0,0 +1,49 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record Job +{ + public required string PackageManager { get; init; } + public AllowedUpdate[]? AllowedUpdates { get; init; } = null; + public bool Debug { get; init; } = false; + public object[]? DependencyGroups { get; init; } = null; + public object[]? Dependencies { get; init; } = null; + public string? DependencyGroupToRefresh { get; init; } = null; + public object[]? ExistingPullRequests { get; init; } = null; + public object[]? ExistingGroupPullRequests { get; init; } = null; + public Dictionary? Experiments { get; init; } = null; + public object[]? IgnoreConditions { get; init; } = null; + public bool LockfileOnly { get; init; } = false; + public string? RequirementsUpdateStrategy { get; init; } = null; + public object[]? SecurityAdvisories { get; init; } = null; + public bool SecurityUpdatesOnly { get; init; } = false; + public required JobSource Source { get; init; } + public bool UpdateSubdependencies { get; init; } = false; + public bool UpdatingAPullRequest { get; init; } = false; + public bool VendorDependencies { get; init; } = false; + public bool RejectExternalCode { get; init; } = false; + public bool RepoPrivate { get; init; } = false; + public object? CommitMessageOptions { get; init; } = null; + public object[]? CredentialsMetadata { get; init; } = null; + public int MaxUpdaterRunTime { get; init; } = 0; + + public IEnumerable GetAllDirectories() + { + var returnedADirectory = false; + if (Source.Directory is not null) + { + returnedADirectory = true; + yield return Source.Directory; + } + + foreach (var directory in Source.Directories ?? []) + { + returnedADirectory = true; + yield return directory; + } + + if (!returnedADirectory) + { + yield return "/"; + } + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobFile.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobFile.cs new file mode 100644 index 0000000000..e193fa9e29 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobFile.cs @@ -0,0 +1,6 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record JobFile +{ + public required Job Job { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobSource.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobSource.cs new file mode 100644 index 0000000000..21f8c17db2 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobSource.cs @@ -0,0 +1,11 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed class JobSource +{ + public required string Provider { get; init; } + public required string Repo { get; init; } + public string? Directory { get; init; } = null; + public string[]? Directories { get; init; } = null; + public string? Hostname { get; init; } = null; + public string? ApiEndpoint { get; init; } = null; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/MarkAsProcessed.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/MarkAsProcessed.cs new file mode 100644 index 0000000000..b29b0be0ab --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/MarkAsProcessed.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record MarkAsProcessed +{ + [JsonPropertyName("base-commit-sha")] + public required string BaseCommitSha { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/ReportedDependency.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/ReportedDependency.cs new file mode 100644 index 0000000000..dc4b673c11 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/ReportedDependency.cs @@ -0,0 +1,16 @@ +using System.Text.Json.Serialization; + +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record ReportedDependency +{ + public required string Name { get; init; } + public required string? Version { get; init; } + public required ReportedRequirement[] Requirements { get; init; } + [JsonPropertyName("previous-version")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PreviousVersion { get; init; } = null; + [JsonPropertyName("previous-requirements")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ReportedRequirement[]? PreviousRequirements { get; init; } = null; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/ReportedRequirement.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/ReportedRequirement.cs new file mode 100644 index 0000000000..5a2d93bb67 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/ReportedRequirement.cs @@ -0,0 +1,9 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record ReportedRequirement +{ + public required string Requirement { get; init; } + public required string File { get; init; } + public string[] Groups { get; init; } = Array.Empty(); + public RequirementSource? Source { get; init; } = null; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/RequirementSource.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/RequirementSource.cs new file mode 100644 index 0000000000..5631b55195 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/RequirementSource.cs @@ -0,0 +1,7 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record RequirementSource +{ + public required string? SourceUrl { get; init; } + public string Type { get; init; } = "nuget_repo"; +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/UpdatedDependencyList.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/UpdatedDependencyList.cs new file mode 100644 index 0000000000..6ecf38e0f0 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/UpdatedDependencyList.cs @@ -0,0 +1,7 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public sealed record UpdatedDependencyList +{ + public required ReportedDependency[] Dependencies { get; init; } + public required string[] DependencyFiles { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/HttpApiHandler.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/HttpApiHandler.cs new file mode 100644 index 0000000000..40d25203d1 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/HttpApiHandler.cs @@ -0,0 +1,59 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +using NuGetUpdater.Core.Run.ApiModel; + +namespace NuGetUpdater.Core.Run; + +public class HttpApiHandler : IApiHandler +{ + private static readonly HttpClient HttpClient = new(); + + public static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + Converters = { new JsonStringEnumConverter() }, + }; + + private readonly string _apiUrl; + private readonly string _jobId; + + public HttpApiHandler(string apiUrl, string jobId) + { + _apiUrl = apiUrl.TrimEnd('/'); + _jobId = jobId; + } + + public async Task UpdateDependencyList(UpdatedDependencyList updatedDependencyList) + { + await PostAsJson("update_dependency_list", updatedDependencyList); + } + + public async Task IncrementMetric(IncrementMetric incrementMetric) + { + await PostAsJson("increment_metric", incrementMetric); + } + + public async Task CreatePullRequest(CreatePullRequest createPullRequest) + { + await PostAsJson("create_pull_request", createPullRequest); + } + + public async Task MarkAsProcessed(MarkAsProcessed markAsProcessed) + { + await PostAsJson("mark_as_processed", markAsProcessed); + } + + private async Task PostAsJson(string endpoint, object body) + { + var wrappedBody = new + { + Data = body, + }; + var payload = JsonSerializer.Serialize(wrappedBody, SerializerOptions); + var content = new StringContent(payload, Encoding.UTF8, "application/json"); + var response = await HttpClient.PostAsync($"{_apiUrl}/update_jobs/{_jobId}/{endpoint}", content); + var _ = response.EnsureSuccessStatusCode(); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs new file mode 100644 index 0000000000..0586795521 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/IApiHandler.cs @@ -0,0 +1,11 @@ +using NuGetUpdater.Core.Run.ApiModel; + +namespace NuGetUpdater.Core.Run; + +public interface IApiHandler +{ + Task UpdateDependencyList(UpdatedDependencyList updatedDependencyList); + Task IncrementMetric(IncrementMetric incrementMetric); + Task CreatePullRequest(CreatePullRequest createPullRequest); + Task MarkAsProcessed(MarkAsProcessed markAsProcessed); +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunResult.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunResult.cs new file mode 100644 index 0000000000..a313c9768c --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunResult.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +using NuGetUpdater.Core.Run.ApiModel; + +namespace NuGetUpdater.Core.Run; + +public sealed record RunResult +{ + [JsonPropertyName("base64_dependency_files")] + public required DependencyFile[] Base64DependencyFiles { get; init; } + [JsonPropertyName("base_commit_sha")] + public required string BaseCommitSha { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs new file mode 100644 index 0000000000..b3b8c4fcc5 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs @@ -0,0 +1,283 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +using NuGetUpdater.Core.Analyze; +using NuGetUpdater.Core.Discover; +using NuGetUpdater.Core.Run.ApiModel; + +namespace NuGetUpdater.Core.Run; + +public class RunWorker +{ + private readonly IApiHandler _apiHandler; + private readonly Logger _logger; + + internal static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.KebabCaseLower, + WriteIndented = true, + Converters = { new JsonStringEnumConverter() }, + }; + + public RunWorker(IApiHandler apiHandler, Logger logger) + { + _apiHandler = apiHandler; + _logger = logger; + } + + public async Task RunAsync(FileInfo jobFilePath, DirectoryInfo repoContentsPath, string baseCommitSha, FileInfo outputFilePath) + { + var jobFileContent = await File.ReadAllTextAsync(jobFilePath.FullName); + var jobWrapper = Deserialize(jobFileContent); + var result = await RunAsync(jobWrapper.Job, repoContentsPath, baseCommitSha); + var resultJson = JsonSerializer.Serialize(result, SerializerOptions); + await File.WriteAllTextAsync(outputFilePath.FullName, resultJson); + } + + public async Task RunAsync(Job job, DirectoryInfo repoContentsPath, string baseCommitSha) + { + MSBuildHelper.RegisterMSBuild(repoContentsPath.FullName, repoContentsPath.FullName); + + var allDependencyFiles = new Dictionary(); + foreach (var directory in job.GetAllDirectories()) + { + var result = await RunForDirectory(job, repoContentsPath, directory, baseCommitSha); + foreach (var dependencyFile in result.Base64DependencyFiles) + { + allDependencyFiles[dependencyFile.Name] = dependencyFile; + } + } + + var runResult = new RunResult() + { + Base64DependencyFiles = allDependencyFiles.Values.ToArray(), + BaseCommitSha = baseCommitSha, + }; + return runResult; + } + + private async Task RunForDirectory(Job job, DirectoryInfo repoContentsPath, string repoDirectory, string baseCommitSha) + { + var discoveryWorker = new DiscoveryWorker(_logger); + var discoveryResult = await discoveryWorker.RunAsync(repoContentsPath.FullName, repoDirectory); + // TODO: check discoveryResult.ErrorType + + _logger.Log("Discovery JSON content:"); + _logger.Log(JsonSerializer.Serialize(discoveryResult, DiscoveryWorker.SerializerOptions)); + + // report dependencies + var discoveredUpdatedDependencies = GetUpdatedDependencyListFromDiscovery(discoveryResult); + await _apiHandler.UpdateDependencyList(discoveredUpdatedDependencies); + + // TODO: pull out relevant dependencies, then check each for updates and track the changes + // TODO: for each top-level dependency, _or_ specific dependency (if security, use transitive) + var originalDependencyFileContents = new Dictionary(); + var allowedUpdates = job.AllowedUpdates ?? []; + var actualUpdatedDependencies = new List(); + if (allowedUpdates.Any(a => a.UpdateType == "all")) + { + await _apiHandler.IncrementMetric(new() + { + Metric = "updater.started", + Tags = { ["operation"] = "group_update_all_versions" }, + }); + + // track original contents for later handling + foreach (var project in discoveryResult.Projects) + { + // TODO: include global.json, etc. + var path = Path.Join(discoveryResult.Path, project.FilePath).NormalizePathToUnix().EnsurePrefix("/"); + var localPath = Path.Join(repoContentsPath.FullName, discoveryResult.Path, project.FilePath); + var content = await File.ReadAllTextAsync(localPath); + originalDependencyFileContents[path] = content; + } + + // do update + _logger.Log($"Running update in directory {repoDirectory}"); + foreach (var project in discoveryResult.Projects) + { + foreach (var dependency in project.Dependencies.Where(d => !d.IsTransitive)) + { + if (dependency.Name == "Microsoft.NET.Sdk") + { + // this can't be updated + // TODO: pull this out of discovery? + continue; + } + + if (dependency.Version is null) + { + // if we don't know the version, there's nothing we can do + continue; + } + + var analyzeWorker = new AnalyzeWorker(_logger); + var dependencyInfo = new DependencyInfo() + { + Name = dependency.Name, + Version = dependency.Version!, + IsVulnerable = false, + IgnoredVersions = [], + Vulnerabilities = [], + }; + var analysisResult = await analyzeWorker.RunAsync(repoContentsPath.FullName, discoveryResult, dependencyInfo); + // TODO: log analysisResult + // TODO: check analysisResult.ErrorType + if (analysisResult.CanUpdate) + { + // TODO: this is inefficient, but not likely causing a bottleneck + var previousDependency = discoveredUpdatedDependencies.Dependencies + .Single(d => d.Name == dependency.Name && d.Requirements.Single().File == Path.Join(discoveryResult.Path, project.FilePath).NormalizePathToUnix().EnsurePrefix("/")); + var updatedDependency = new ReportedDependency() + { + Name = dependency.Name, + Version = analysisResult.UpdatedVersion, + Requirements = + [ + new ReportedRequirement() + { + File = Path.Join(discoveryResult.Path, project.FilePath).NormalizePathToUnix().EnsurePrefix("/"), + Requirement = analysisResult.UpdatedVersion, + Groups = previousDependency.Requirements.Single().Groups, + Source = new RequirementSource() + { + SourceUrl = analysisResult.UpdatedDependencies.Single(d => d.Name == dependency.Name).InfoUrl, + }, + } + ], + PreviousVersion = dependency.Version, + PreviousRequirements = previousDependency.Requirements, + }; + + var updateWorker = new UpdaterWorker(_logger); + var dependencyFilePath = Path.Join(discoveryResult.Path, project.FilePath).NormalizePathToUnix(); + var updateResult = await updateWorker.RunAsync(repoContentsPath.FullName, dependencyFilePath, dependency.Name, dependency.Version!, analysisResult.UpdatedVersion, isTransitive: false); + // TODO: check specific contents of result.ErrorType + // TODO: need to report if anything was actually updated + if (updateResult.ErrorType is null || updateResult.ErrorType == ErrorType.None) + { + actualUpdatedDependencies.Add(updatedDependency); + } + } + } + } + + // create PR - we need to manually check file contents; we can't easily use `git status` in tests + var updatedDependencyFiles = new List(); + foreach (var project in discoveryResult.Projects) + { + var path = Path.Join(discoveryResult.Path, project.FilePath).NormalizePathToUnix().EnsurePrefix("/"); + var localPath = Path.Join(repoContentsPath.FullName, discoveryResult.Path, project.FilePath); + var updatedContent = await File.ReadAllTextAsync(localPath); + var originalContent = originalDependencyFileContents[path]; + if (updatedContent != originalContent) + { + updatedDependencyFiles.Add(new DependencyFile() + { + Name = project.FilePath, + Content = updatedContent, + Directory = discoveryResult.Path, + }); + } + } + + if (updatedDependencyFiles.Count > 0) + { + var createPullRequest = new CreatePullRequest() + { + Dependencies = actualUpdatedDependencies.ToArray(), + UpdatedDependencyFiles = updatedDependencyFiles.ToArray(), + BaseCommitSha = baseCommitSha, + CommitMessage = "TODO: message", + PrTitle = "TODO: title", + PrBody = "TODO: body", + }; + await _apiHandler.CreatePullRequest(createPullRequest); + // TODO: log updated dependencies to console + } + else + { + // TODO: log or throw if nothing was updated, but was expected to be + } + } + else + { + // TODO: throw if no updates performed + } + + await _apiHandler.MarkAsProcessed(new() { BaseCommitSha = baseCommitSha }); + var result = new RunResult() + { + Base64DependencyFiles = originalDependencyFileContents.Select(kvp => new DependencyFile() + { + Name = Path.GetFileName(kvp.Key), + Content = Convert.ToBase64String(Encoding.UTF8.GetBytes(kvp.Value)), + Directory = Path.GetDirectoryName(kvp.Key)!.NormalizePathToUnix(), + }).ToArray(), + BaseCommitSha = baseCommitSha, + }; + return result; + } + + internal static UpdatedDependencyList GetUpdatedDependencyListFromDiscovery(WorkspaceDiscoveryResult discoveryResult) + { + string GetFullRepoPath(string path) + { + // ensures `path\to\file` is `/path/to/file` + return Path.Join(discoveryResult.Path, path).NormalizePathToUnix().NormalizeUnixPathParts().EnsurePrefix("/"); + } + + var auxiliaryFiles = new List(); + if (discoveryResult.GlobalJson is not null) + { + auxiliaryFiles.Add(GetFullRepoPath(discoveryResult.GlobalJson.FilePath)); + } + if (discoveryResult.DotNetToolsJson is not null) + { + auxiliaryFiles.Add(GetFullRepoPath(discoveryResult.DotNetToolsJson.FilePath)); + } + if (discoveryResult.DirectoryPackagesProps is not null) + { + auxiliaryFiles.Add(GetFullRepoPath(discoveryResult.DirectoryPackagesProps.FilePath)); + } + + var updatedDependencyList = new UpdatedDependencyList() + { + Dependencies = discoveryResult.Projects.SelectMany(p => + { + return p.Dependencies.Where(d => d.Version is not null).Select(d => + new ReportedDependency() + { + Name = d.Name, + Requirements = d.IsTransitive ? [] : [new ReportedRequirement() + { + File = GetFullRepoPath(p.FilePath), + Requirement = d.Version!, + Groups = ["dependencies"], + }], + Version = d.Version, + } + ); + }).ToArray(), + DependencyFiles = discoveryResult.Projects.Select(p => GetFullRepoPath(p.FilePath)).Concat(auxiliaryFiles).ToArray(), + }; + return updatedDependencyList; + } + + public static JobFile Deserialize(string json) + { + var jobFile = JsonSerializer.Deserialize(json, SerializerOptions); + if (jobFile is null) + { + throw new InvalidOperationException("Unable to deserialize job wrapper."); + } + + if (jobFile.Job.PackageManager != "nuget") + { + throw new InvalidOperationException("Package manager must be 'nuget'"); + } + + return jobFile; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs index c348d30a0f..db0f3b7fd8 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs @@ -24,6 +24,15 @@ public UpdaterWorker(Logger logger) } public async Task RunAsync(string repoRootPath, string workspacePath, string dependencyName, string previousDependencyVersion, string newDependencyVersion, bool isTransitive, string? resultOutputPath = null) + { + var result = await RunAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); + if (resultOutputPath is { }) + { + await WriteResultFile(result, resultOutputPath, _logger); + } + } + + public async Task RunAsync(string repoRootPath, string workspacePath, string dependencyName, string previousDependencyVersion, string newDependencyVersion, bool isTransitive) { MSBuildHelper.RegisterMSBuild(Environment.CurrentDirectory, repoRootPath); UpdateOperationResult result; @@ -83,10 +92,7 @@ public async Task RunAsync(string repoRootPath, string workspacePath, string dep } _processedProjectPaths.Clear(); - if (resultOutputPath is { }) - { - await WriteResultFile(result, resultOutputPath, _logger); - } + return result; } internal static async Task WriteResultFile(UpdateOperationResult result, string resultOutputPath, Logger logger) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/PathHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/PathHelper.cs index 3810795c3a..8d680cb991 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/PathHelper.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/PathHelper.cs @@ -25,8 +25,42 @@ public static string JoinPath(string? path1, string path2) : Path.Combine(path1, path2); } + public static string EnsurePrefix(this string s, string prefix) => s.StartsWith(prefix) ? s : prefix + s; + public static string NormalizePathToUnix(this string path) => path.Replace("\\", "/"); + public static string NormalizeUnixPathParts(this string path) + { + var parts = path.Split('/'); + var resultantParts = new List(); + foreach (var part in parts) + { + switch (part) + { + case "": + case ".": + break; + case "..": + if (resultantParts.Count > 0) + { + resultantParts.RemoveAt(resultantParts.Count - 1); + } + break; + default: + resultantParts.Add(part); + break; + } + } + + var result = string.Join("/", resultantParts); + if (path.StartsWith("/") && !result.StartsWith("/")) + { + result = "/" + result; + } + + return result; + } + public static string GetFullPathFromRelative(string rootPath, string relativePath) => Path.GetFullPath(JoinPath(rootPath, relativePath.NormalizePathToUnix())); diff --git a/nuget/script/run b/nuget/script/run new file mode 100644 index 0000000000..eaea942642 --- /dev/null +++ b/nuget/script/run @@ -0,0 +1,11 @@ +#!/bin/bash +# shellcheck disable=all + +nuget_experiment_value=$(cat "$DEPENDABOT_JOB_PATH" | jq '.job.experiments.nuget_native_updater') +echo "NuGet native updater experiment value: $nuget_experiment_value" + +if echo "$nuget_experiment_value" | grep -q 'true'; then + pwsh "$DEPENDABOT_HOME/dependabot-updater/bin/main.ps1" $* +else + "$DEPENDABOT_HOME/dependabot-updater/bin/run-original" $* +fi diff --git a/nuget/updater/main.ps1 b/nuget/updater/main.ps1 new file mode 100644 index 0000000000..e20b22d15a --- /dev/null +++ b/nuget/updater/main.ps1 @@ -0,0 +1,96 @@ +Set-StrictMode -version 2.0 +$ErrorActionPreference = "Stop" + +$jobString = Get-Content -Path $env:DEPENDABOT_JOB_PATH +$job = (ConvertFrom-Json -InputObject $jobString).job + +function Get-Files { + Write-Host "Job: $($job | ConvertTo-Json)" + $sourceRepo = $job.source.repo + # TODO: handle other values from $job.source.provider + $url = "https://github.com/$sourceRepo" + $path = $env:DEPENDABOT_REPO_CONTENTS_PATH + $cloneOptions = "--no-tags --depth 1 --recurse-submodules --shallow-submodules" + if ("branch" -in $job.source.PSobject.Properties.Name) { + $cloneOptions += " --branch $($job.source.branch) --single-branch" + } + + Invoke-Expression "git clone $cloneOptions $url $path" + + if ("commit" -in $job.source.PSobject.Properties.Name) { + # this is only called for testing; production will never pass a commit + Push-Location $path + + $fetchOptions = "--depth 1 --recurse-submodules=on-demand" + Invoke-Expression "git fetch $fetchOptions origin $($job.source.commit)" + + $resetOptions = "--hard --recurse-submodules" + Invoke-Expression "git reset $resetOptions $($job.source.commit)" + + Pop-Location + } +} + +function Install-Sdks([string] $directory) { + $installedSdks = dotnet --list-sdks | ForEach-Object { $_.Split(' ')[0] } + if ($installedSdks.GetType().Name -eq "String") { + # if only a single value was returned (expected in the container), then force it to an array + $installedSdks = @($installedSdks) + } + Write-Host "Currently installed SDKs: $installedSdks" + $rootDir = Convert-Path $env:DEPENDABOT_REPO_CONTENTS_PATH + $candidateDir = Convert-Path "$rootDir/$directory" + while ($true) { + $globalJsonPath = Join-Path $candidateDir "global.json" + if (Test-Path $globalJsonPath) { + $globalJson = Get-Content $globalJsonPath | ConvertFrom-Json + $sdkVersion = $globalJson.sdk.version + if (-Not ($sdkVersion -in $installedSdks)) { + $installedSdks += $sdkVersion + Write-Host "Installing SDK $sdkVersion as specified in $globalJsonPath" + & $env:DOTNET_INSTALL_SCRIPT_PATH --version $sdkVersion --install-dir $env:DOTNET_INSTALL_DIR + } + } + + $candidateDir = Split-Path -Parent $candidateDir + if ($candidateDir -eq $rootDir) { + break + } + } + + # report the final set + dotnet --list-sdks +} + +function Update-Files { + # install relevant SDKs + Install-Sdks $job.source.directory + # TODO: install workloads? + + Push-Location $env:DEPENDABOT_REPO_CONTENTS_PATH + $baseCommitSha = git rev-parse HEAD + Pop-Location + + $updaterTool = "$env:DEPENDABOT_NATIVE_HELPERS_PATH/nuget/NuGetUpdater/NuGetUpdater.Cli" + & $updaterTool run ` + --job-path $env:DEPENDABOT_JOB_PATH ` + --repo-contents-path $env:DEPENDABOT_REPO_CONTENTS_PATH ` + --api-url $env:DEPENDABOT_API_URL ` + --job-id $env:DEPENDABOT_JOB_ID ` + --output-path $env:DEPENDABOT_OUTPUT_PATH ` + --base-commit-sha $baseCommitSha ` + --verbose +} + +try { + Switch ($args[0]) { + "fetch_files" { Get-Files } + "update_files" { Update-Files } + } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 +}