Merge pull request #204 from brminnick/Move-NuGet-Service-to-Cloud

Move NuGet Service to cloud

GitTrends.Functions/Functions/GetLibraries.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using GitTrends.Shared;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Azure.WebJobs.Extensions.Http;
+using Microsoft.Extensions.Logging;
+
+namespace GitTrends.Functions
+{
+    class GetLibraries
+    {
+        readonly BlobStorageService _blobStorageService;
+
+        public GetLibraries(BlobStorageService blobStorageService) => _blobStorageService = blobStorageService;
+
+        [FunctionName(nameof(GetLibraries))]
+        public Task<IReadOnlyList<NuGetPackageModel>> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get")] HttpRequest request, ILogger log) => _blobStorageService.GetNuGetLibraries();
+    }
+}

GitTrends.Functions/Functions/UpdateLibraryCache.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using GitTrends.Shared;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+
+namespace GitTrends.Functions
+{
+    class UpdateLibraryCache
+    {
+        readonly NuGetService _nuGetService;
+        readonly BlobStorageService _blobStorageService;
+
+        public UpdateLibraryCache(NuGetService nuGetService, BlobStorageService blobStorageService) =>
+            (_nuGetService, _blobStorageService) = (nuGetService, blobStorageService);
+
+        [FunctionName(nameof(UpdateLibraryCache))]
+        public async Task Run([TimerTrigger("0 0 0 * * *")] TimerInfo myTimer, ILogger log)
+        {
+            var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
+
+            log.LogInformation("Retrieving NuGet Packages");
+
+            var nugetPackageDictionary = new Dictionary<string, (Uri IconUri, Uri NugetUri)>();
+            await foreach (var (Title, ImageUri, NugetUri) in _nuGetService.GetPackageInfo(cancellationTokenSource.Token))
+            {
+                log.LogInformation($"Found {Title}");
+                if (!nugetPackageDictionary.ContainsKey(Title))
+                    nugetPackageDictionary.Add(Title, (ImageUri, NugetUri));
+            }
+
+            var nugetPackageModelList = new List<NuGetPackageModel>();
+            foreach (var nugetPackage in nugetPackageDictionary)
+                nugetPackageModelList.Add(new NuGetPackageModel(nugetPackage.Key, nugetPackage.Value.IconUri, nugetPackage.Value.NugetUri));
+
+            var blobName = $"Libraries_{DateTime.UtcNow:o}.json";
+            log.LogInformation($"Saving NuGet Pacakges to Blob Storage: {blobName}");
+
+            await _blobStorageService.UploadNuGetLibraries(nugetPackageModelList, blobName).ConfigureAwait(false);
+        }
+    }
+}

GitTrends.Functions/GitTrends.Functions.csproj

@@ -27,6 +27,7 @@
         <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="3.1.11" />
         <PackageReference Include="Microsoft.Azure.NotificationHubs" Version="4.1.0" />
         <PackageReference Include="Xamarin.Essentials.Interfaces" Version="1.6.1" />
+        <PackageReference Include="NuGet.Protocol" Version="5.8.1" />
     </ItemGroup>
     <ItemGroup>
         <None Update="host.json">

GitTrends.Functions/Properties/ServiceDependencies/GitTrendsFunctions - Zip Deploy/profile.arm.json

@@ -0,0 +1,175 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "metadata": {
+    "_dependencyType": "function.windows.appService"
+  },
+  "parameters": {
+    "resourceGroupName": {
+      "type": "string",
+      "defaultValue": "GitTrends",
+      "metadata": {
+        "description": "Name of the resource group for the resource. It is recommended to put resources under same resource group for better tracking."
+      }
+    },
+    "resourceGroupLocation": {
+      "type": "string",
+      "defaultValue": "westus",
+      "metadata": {
+        "description": "Location of the resource group. Resource groups could have different location than resources, however by default we use API versions from latest hybrid profile which support all locations for resource types we support."
+      }
+    },
+    "resourceName": {
+      "type": "string",
+      "defaultValue": "GitTrendsFunctions",
+      "metadata": {
+        "description": "Name of the main resource to be created by this template."
+      }
+    },
+    "resourceLocation": {
+      "type": "string",
+      "defaultValue": "[parameters('resourceGroupLocation')]",
+      "metadata": {
+        "description": "Location of the resource. By default use resource group's location, unless the resource provider is not supported there."
+      }
+    }
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Resources/resourceGroups",
+      "name": "[parameters('resourceGroupName')]",
+      "location": "[parameters('resourceGroupLocation')]",
+      "apiVersion": "2019-10-01"
+    },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "name": "[concat(parameters('resourceGroupName'), 'Deployment', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId)))]",
+      "resourceGroup": "[parameters('resourceGroupName')]",
+      "apiVersion": "2019-10-01",
+      "dependsOn": [
+        "[parameters('resourceGroupName')]"
+      ],
+      "properties": {
+        "mode": "Incremental",
+        "expressionEvaluationOptions": {
+          "scope": "inner"
+        },
+        "parameters": {
+          "resourceGroupName": {
+            "value": "[parameters('resourceGroupName')]"
+          },
+          "resourceGroupLocation": {
+            "value": "[parameters('resourceGroupLocation')]"
+          },
+          "resourceName": {
+            "value": "[parameters('resourceName')]"
+          },
+          "resourceLocation": {
+            "value": "[parameters('resourceLocation')]"
+          }
+        },
+        "template": {
+          "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+          "contentVersion": "1.0.0.0",
+          "parameters": {
+            "resourceGroupName": {
+              "type": "string"
+            },
+            "resourceGroupLocation": {
+              "type": "string"
+            },
+            "resourceName": {
+              "type": "string"
+            },
+            "resourceLocation": {
+              "type": "string"
+            }
+          },
+          "variables": {
+            "storage_name": "[toLower(concat('storage', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId))))]",
+            "appServicePlan_name": "[concat('Plan', uniqueString(concat(parameters('resourceName'), subscription().subscriptionId)))]",
+            "storage_ResourceId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Storage/storageAccounts/', variables('storage_name'))]",
+            "appServicePlan_ResourceId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Web/serverFarms/', variables('appServicePlan_name'))]",
+            "function_ResourceId": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('resourceGroupName'), '/providers/Microsoft.Web/sites/', parameters('resourceName'))]"
+          },
+          "resources": [
+            {
+              "location": "[parameters('resourceLocation')]",
+              "name": "[parameters('resourceName')]",
+              "type": "Microsoft.Web/sites",
+              "apiVersion": "2015-08-01",
+              "tags": {
+                "[concat('hidden-related:', variables('appServicePlan_ResourceId'))]": "empty"
+              },
+              "dependsOn": [
+                "[variables('appServicePlan_ResourceId')]",
+                "[variables('storage_ResourceId')]"
+              ],
+              "kind": "functionapp",
+              "properties": {
+                "name": "[parameters('resourceName')]",
+                "kind": "functionapp",
+                "httpsOnly": true,
+                "reserved": false,
+                "serverFarmId": "[variables('appServicePlan_ResourceId')]",
+                "siteConfig": {
+                  "alwaysOn": true
+                }
+              },
+              "identity": {
+                "type": "SystemAssigned"
+              },
+              "resources": [
+                {
+                  "name": "appsettings",
+                  "type": "config",
+                  "apiVersion": "2015-08-01",
+                  "dependsOn": [
+                    "[variables('function_ResourceId')]"
+                  ],
+                  "properties": {
+                    "AzureWebJobsDashboard": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storage_name'), ';AccountKey=', listKeys(variables('storage_ResourceId'), '2017-10-01').keys[0].value, ';EndpointSuffix=', 'core.windows.net')]",
+                    "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storage_name'), ';AccountKey=', listKeys(variables('storage_ResourceId'), '2017-10-01').keys[0].value, ';EndpointSuffix=', 'core.windows.net')]",
+                    "FUNCTIONS_EXTENSION_VERSION": "~3",
+                    "FUNCTIONS_WORKER_RUNTIME": "dotnet"
+                  }
+                }
+              ]
+            },
+            {
+              "location": "[parameters('resourceGroupLocation')]",
+              "name": "[variables('storage_name')]",
+              "type": "Microsoft.Storage/storageAccounts",
+              "apiVersion": "2017-10-01",
+              "tags": {
+                "[concat('hidden-related:', concat('/providers/Microsoft.Web/sites/', parameters('resourceName')))]": "empty"
+              },
+              "properties": {
+                "supportsHttpsTrafficOnly": true
+              },
+              "sku": {
+                "name": "Standard_LRS"
+              },
+              "kind": "Storage"
+            },
+            {
+              "location": "[parameters('resourceGroupLocation')]",
+              "name": "[variables('appServicePlan_name')]",
+              "type": "Microsoft.Web/serverFarms",
+              "apiVersion": "2015-08-01",
+              "sku": {
+                "name": "S1",
+                "tier": "Standard",
+                "family": "S",
+                "size": "S1"
+              },
+              "properties": {
+                "name": "[variables('appServicePlan_name')]"
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

GitTrends.Functions/Services/BlobStorageService.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Newtonsoft.Json;
+using GitTrends.Shared;
+
+namespace GitTrends.Functions
+{
+    class BlobStorageService
+    {
+        const string _libraryContainerName = "librarycache";
+        readonly CloudBlobClient _blobClient;
+
+        public BlobStorageService(CloudBlobClient cloudBlobClient) => _blobClient = cloudBlobClient;
+
+        public Task UploadNuGetLibraries(IEnumerable<NuGetPackageModel> nuGetPackageModels, string blobName)
+        {
+            var container = GetBlobContainer(_libraryContainerName);
+            container.CreateIfNotExistsAsync().ConfigureAwait(false);
+
+            var blob = container.GetBlockBlobReference(blobName);
+            return blob.UploadTextAsync(JsonConvert.SerializeObject(nuGetPackageModels));
+        }
+
+        public async Task<IReadOnlyList<NuGetPackageModel>> GetNuGetLibraries()
+        {
+            var blobList = new List<CloudBlockBlob>();
+            await foreach (var blob in GetBlobs<CloudBlockBlob>(_libraryContainerName).ConfigureAwait(false))
+            {
+                blobList.Add(blob);
+            }
+
+            var nugetPackageModelBlob = blobList.OrderByDescending(x => x.Properties.Created).First();
+            var serializedNuGetPackageList = await nugetPackageModelBlob.DownloadTextAsync().ConfigureAwait(false);
+
+            return JsonConvert.DeserializeObject<IReadOnlyList<NuGetPackageModel>>(serializedNuGetPackageList) ?? throw new NullReferenceException();
+        }
+
+        async IAsyncEnumerable<T> GetBlobs<T>(string containerName, string prefix = "", int? maxresultsPerQuery = null, BlobListingDetails blobListingDetails = BlobListingDetails.None) where T : ICloudBlob
+        {
+            var blobContainer = GetBlobContainer(containerName);
+
+            BlobContinuationToken? continuationToken = null;
+
+            do
+            {
+                var response = await blobContainer.ListBlobsSegmentedAsync(prefix, true, blobListingDetails, maxresultsPerQuery, continuationToken, null, null).ConfigureAwait(false);
+                continuationToken = response?.ContinuationToken;
+
+                var blobListFromResponse = response?.Results?.OfType<T>() ?? Enumerable.Empty<T>();
+
+                foreach (var blob in blobListFromResponse)
+                {
+                    yield return blob;
+                }
+
+            } while (continuationToken != null);
+
+        }
+
+        CloudBlobContainer GetBlobContainer(string containerName) => _blobClient.GetContainerReference(containerName);
+    }
+}

GitTrends.Functions/Services/GitHubApiV3Service.cs

@@ -13,5 +13,8 @@ class GitHubApiV3Service : BaseApiService
 
         public Task<HttpResponseMessage> GetGitHubApiResponse(string token, CancellationToken cancellationToken) =>
             AttemptAndRetry(() => _gitHubApiV3Client.GetGitHubApiResponse_Authenticated($"Bearer {token}"), cancellationToken);
+
+        public Task<RepositoryFile> GetGitTrendsFile(string fileName, CancellationToken cancellationToken) =>
+            AttemptAndRetry(() => _gitHubApiV3Client.GetGitTrendsFile(fileName), cancellationToken);
     }
 }