From 8d7856f4c7c369df3a47c464007442910fa81d97 Mon Sep 17 00:00:00 2001 From: Patrik Svensson Date: Wed, 24 Feb 2021 10:45:41 +0100 Subject: [PATCH 1/4] Adds support for NuGet V2 upstream package sources --- src/BaGet.Core/BaGet.Core.csproj | 1 + src/BaGet.Core/Configuration/MirrorOptions.cs | 5 + .../DependencyInjectionExtensions.cs | 13 +- .../Mirror/Clients/MirrorV2Client.cs | 114 ++++++++++++++++++ .../Mirror/Clients/MirrorV3Client.cs | 36 ++++++ src/BaGet.Core/Mirror/IMirrorNuGetClient.cs | 16 +++ .../Mirror/MirrorNuGetClientAdapter.cs | 8 ++ src/BaGet.Core/Mirror/MirrorService.cs | 17 ++- src/BaGet/appsettings.json | 2 + .../Mirror/MirrorServiceTests.cs | 2 +- 10 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 src/BaGet.Core/Mirror/Clients/MirrorV2Client.cs create mode 100644 src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs create mode 100644 src/BaGet.Core/Mirror/IMirrorNuGetClient.cs create mode 100644 src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs diff --git a/src/BaGet.Core/BaGet.Core.csproj b/src/BaGet.Core/BaGet.Core.csproj index 8c1287492..5c3bf1b66 100644 --- a/src/BaGet.Core/BaGet.Core.csproj +++ b/src/BaGet.Core/BaGet.Core.csproj @@ -13,6 +13,7 @@ + diff --git a/src/BaGet.Core/Configuration/MirrorOptions.cs b/src/BaGet.Core/Configuration/MirrorOptions.cs index f5bb29467..09db895d3 100644 --- a/src/BaGet.Core/Configuration/MirrorOptions.cs +++ b/src/BaGet.Core/Configuration/MirrorOptions.cs @@ -17,6 +17,11 @@ public class MirrorOptions : IValidatableObject /// public Uri PackageSource { get; set; } + /// + /// Whether or not the package source is a v2 package source feed. + /// + public bool Legacy { get; set; } + /// /// The time before a download from the package source times out. /// diff --git a/src/BaGet.Core/Extensions/DependencyInjectionExtensions.cs b/src/BaGet.Core/Extensions/DependencyInjectionExtensions.cs index 7cf4650de..a4538920a 100644 --- a/src/BaGet.Core/Extensions/DependencyInjectionExtensions.cs +++ b/src/BaGet.Core/Extensions/DependencyInjectionExtensions.cs @@ -99,11 +99,14 @@ private static void AddBaGetServices(this IServiceCollection services) services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); + services.TryAddTransient(); services.TryAddTransient(); services.TryAddSingleton(); services.TryAddTransient(); services.TryAddTransient(IMirrorServiceFactory); + services.TryAddTransient(IMirrorNuGetClientFactory); } private static void AddDefaultProviders(this IServiceCollection services) @@ -195,8 +198,16 @@ private static IMirrorService IMirrorServiceFactory(IServiceProvider provider) { var options = provider.GetRequiredService>(); var service = options.Value.Enabled ? typeof(MirrorService) : typeof(NullMirrorService); - + return (IMirrorService)provider.GetRequiredService(service); } + + private static IMirrorNuGetClient IMirrorNuGetClientFactory(IServiceProvider provider) + { + var options = provider.GetRequiredService>(); + var service = options.Value.Legacy ? typeof(MirrorV2Client) : typeof(MirrorV3Client); + + return (IMirrorNuGetClient)provider.GetRequiredService(service); + } } } diff --git a/src/BaGet.Core/Mirror/Clients/MirrorV2Client.cs b/src/BaGet.Core/Mirror/Clients/MirrorV2Client.cs new file mode 100644 index 000000000..b5634db56 --- /dev/null +++ b/src/BaGet.Core/Mirror/Clients/MirrorV2Client.cs @@ -0,0 +1,114 @@ +using BaGet.Protocol.Models; +using Microsoft.Extensions.Options; +using NuGet.Common; +using NuGet.Configuration; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace BaGet.Core +{ + internal sealed class MirrorV2Client : IMirrorNuGetClient + { + private readonly ILogger _logger; + private readonly SourceCacheContext _cache; + private readonly SourceRepository _repository; + + public MirrorV2Client(IOptionsSnapshot options) + { + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (options.Value?.PackageSource?.AbsolutePath == null) + { + throw new ArgumentException("No mirror package source has been set."); + } + + _logger = NullLogger.Instance; + _cache = new SourceCacheContext(); + _repository = Repository.Factory.GetCoreV2(new PackageSource(options.Value.PackageSource.AbsoluteUri)); + } + + public async Task> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken) + { + var resource = await _repository.GetResourceAsync(); + var versions = await resource.GetAllVersionsAsync(id, _cache, _logger, cancellationToken); + + return versions.ToList(); + } + + public async Task> GetPackageMetadataAsync(string id, CancellationToken cancellationToken) + { + var resource = await _repository.GetResourceAsync(); + var packages = await resource.GetMetadataAsync(id, includePrerelease: true, includeUnlisted: false, _cache, _logger, cancellationToken); + + var result = new List(); + foreach (var package in packages) + { + result.Add(new PackageMetadata + { + Authors = package.Authors, + Description = package.Description, + IconUrl = package.IconUrl?.AbsoluteUri, + LicenseUrl = package.LicenseUrl?.AbsoluteUri, + Listed = package.IsListed, + PackageId = id, + Summary = package.Summary, + Version = package.Identity.Version.ToString(), + Tags = package.Tags?.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries), + Title = package.Title, + RequireLicenseAcceptance = package.RequireLicenseAcceptance, + Published = package.Published?.UtcDateTime ?? DateTimeOffset.MinValue, + ProjectUrl = package.ProjectUrl?.AbsoluteUri, + DependencyGroups = GetDependencies(package), + }); + } + + return result; + } + + public async Task DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken) + { + var packageStream = new MemoryStream(); + var resource = await _repository.GetResourceAsync(); + await resource.CopyNupkgToStreamAsync(id, version, packageStream, _cache, _logger, cancellationToken); + packageStream.Seek(0, SeekOrigin.Begin); + + return packageStream; + } + + private IReadOnlyList GetDependencies(IPackageSearchMetadata package) + { + var groupItems = new List(); + foreach (var set in package.DependencySets) + { + var item = new DependencyGroupItem + { + TargetFramework = set.TargetFramework.Framework, + Dependencies = new List() + }; + + foreach (var dependency in set.Packages) + { + item.Dependencies.Add(new DependencyItem + { + Id = dependency.Id, + Range = dependency.VersionRange.ToNormalizedString(), + }); + } + + groupItems.Add(item); + } + + return groupItems; + } + } +} diff --git a/src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs b/src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs new file mode 100644 index 000000000..3dbae0015 --- /dev/null +++ b/src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs @@ -0,0 +1,36 @@ +using BaGet.Protocol; +using BaGet.Protocol.Models; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace BaGet.Core +{ + internal sealed class MirrorV3Client : IMirrorNuGetClient + { + private readonly NuGetClient _client; + + public MirrorV3Client(NuGetClient client) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public Task DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken) + { + return _client.DownloadPackageAsync(id, version, cancellationToken); + } + + public Task> GetPackageMetadataAsync(string id, CancellationToken cancellationToken) + { + return _client.GetPackageMetadataAsync(id, cancellationToken); + } + + public Task> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken) + { + return _client.ListPackageVersionsAsync(id, includeUnlisted, cancellationToken); + } + } +} diff --git a/src/BaGet.Core/Mirror/IMirrorNuGetClient.cs b/src/BaGet.Core/Mirror/IMirrorNuGetClient.cs new file mode 100644 index 000000000..4b51d088c --- /dev/null +++ b/src/BaGet.Core/Mirror/IMirrorNuGetClient.cs @@ -0,0 +1,16 @@ +using BaGet.Protocol.Models; +using NuGet.Versioning; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace BaGet.Core +{ + public interface IMirrorNuGetClient + { + Task> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken); + Task> GetPackageMetadataAsync(string id, CancellationToken cancellationToken); + Task DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken); + } +} diff --git a/src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs b/src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs new file mode 100644 index 000000000..369dd9373 --- /dev/null +++ b/src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs @@ -0,0 +1,8 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BaGet.Core.Mirror +{ + +} diff --git a/src/BaGet.Core/Mirror/MirrorService.cs b/src/BaGet.Core/Mirror/MirrorService.cs index 88bebadc7..2bafa516c 100644 --- a/src/BaGet.Core/Mirror/MirrorService.cs +++ b/src/BaGet.Core/Mirror/MirrorService.cs @@ -16,13 +16,13 @@ namespace BaGet.Core public class MirrorService : IMirrorService { private readonly IPackageService _localPackages; - private readonly NuGetClient _upstreamClient; + private readonly IMirrorNuGetClient _upstreamClient; private readonly IPackageIndexingService _indexer; private readonly ILogger _logger; public MirrorService( IPackageService localPackages, - NuGetClient upstreamClient, + IMirrorNuGetClient upstreamClient, IPackageIndexingService indexer, ILogger logger) { @@ -32,6 +32,19 @@ public MirrorService( _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } + public static MirrorService Create( + IPackageService localPackages, + NuGetClient client, + IPackageIndexingService indexer, + ILogger logger) + { + return new MirrorService( + localPackages, + new MirrorV3Client(client), + indexer, + logger); + } + public async Task> FindPackageVersionsOrNullAsync( string id, CancellationToken cancellationToken) diff --git a/src/BaGet/appsettings.json b/src/BaGet/appsettings.json index 3c3ee86f2..d2ea0711a 100644 --- a/src/BaGet/appsettings.json +++ b/src/BaGet/appsettings.json @@ -19,6 +19,8 @@ "Mirror": { "Enabled": false, + // Uncomment this to use the NuGet v2 protocol + //"Legacy": true, "PackageSource": "https://api.nuget.org/v3/index.json" }, diff --git a/tests/BaGet.Core.Tests/Mirror/MirrorServiceTests.cs b/tests/BaGet.Core.Tests/Mirror/MirrorServiceTests.cs index 0a10afd43..dc567431b 100644 --- a/tests/BaGet.Core.Tests/Mirror/MirrorServiceTests.cs +++ b/tests/BaGet.Core.Tests/Mirror/MirrorServiceTests.cs @@ -61,7 +61,7 @@ public FactsBase() _upstream = new Mock(); _indexer = new Mock(); - _target = new MirrorService( + _target = MirrorService.Create( _packages.Object, _upstream.Object, _indexer.Object, From ca2d5d0e03917ed78f84410478f6c452aeebb803 Mon Sep 17 00:00:00 2001 From: Patrik Svensson Date: Thu, 25 Feb 2021 10:23:32 +0100 Subject: [PATCH 2/4] Remove NuGet.Packaging reference --- src/BaGet.Core/BaGet.Core.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/BaGet.Core/BaGet.Core.csproj b/src/BaGet.Core/BaGet.Core.csproj index 5c3bf1b66..4b5cc093e 100644 --- a/src/BaGet.Core/BaGet.Core.csproj +++ b/src/BaGet.Core/BaGet.Core.csproj @@ -12,7 +12,6 @@ - From 101de99ecd03909919d78f6a15ea4ce2b7b258a9 Mon Sep 17 00:00:00 2001 From: Patrik Svensson Date: Thu, 25 Feb 2021 10:24:04 +0100 Subject: [PATCH 3/4] Await async tasks instead of returning Task --- src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs b/src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs index 3dbae0015..09e928cc2 100644 --- a/src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs +++ b/src/BaGet.Core/Mirror/Clients/MirrorV3Client.cs @@ -18,19 +18,19 @@ public MirrorV3Client(NuGetClient client) _client = client ?? throw new ArgumentNullException(nameof(client)); } - public Task DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken) + public async Task DownloadPackageAsync(string id, NuGetVersion version, CancellationToken cancellationToken) { - return _client.DownloadPackageAsync(id, version, cancellationToken); + return await _client.DownloadPackageAsync(id, version, cancellationToken); } - public Task> GetPackageMetadataAsync(string id, CancellationToken cancellationToken) + public async Task> GetPackageMetadataAsync(string id, CancellationToken cancellationToken) { - return _client.GetPackageMetadataAsync(id, cancellationToken); + return await _client.GetPackageMetadataAsync(id, cancellationToken); } - public Task> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken) + public async Task> ListPackageVersionsAsync(string id, bool includeUnlisted, CancellationToken cancellationToken) { - return _client.ListPackageVersionsAsync(id, includeUnlisted, cancellationToken); + return await _client.ListPackageVersionsAsync(id, includeUnlisted, cancellationToken); } } } From 8dff49da0c5b8320d25581f7404accc883771cdc Mon Sep 17 00:00:00 2001 From: Patrik Svensson Date: Thu, 25 Feb 2021 10:24:20 +0100 Subject: [PATCH 4/4] Remove MirrorNuGetClientAdapter --- src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs diff --git a/src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs b/src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs deleted file mode 100644 index 369dd9373..000000000 --- a/src/BaGet.Core/Mirror/MirrorNuGetClientAdapter.cs +++ /dev/null @@ -1,8 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace BaGet.Core.Mirror -{ - -}