EES-4856 Changes in response to comments

Author: mmyoungman

src/GovUk.Education.ExploreEducationStatistics.Admin/Controllers/Api/Bau/DataSetFileMetaMigrationController.cs

@@ -54,7 +54,7 @@ public async Task<DataSetFileMetaMigrationResult> MigrateReleaseSeries(
                     f.DataSetFileMeta != null
                     && f.Type == FileType.Data)
                 .ToListAsync(cancellationToken: cancellationToken))
-            .Where(f => f.DataSetFileMeta.TimeIdentifier != null);
+            .Where(f => f.DataSetFileMeta.Years != null);
 
         if (num != null)
         {

src/GovUk.Education.ExploreEducationStatistics.Content.Api.Tests/Controllers/DataSetFilesControllerTests.cs

@@ -1859,9 +1859,14 @@ public async Task FetchDataSetDetails_Success()
                     .WithPublicApiDataSetVersion(major: 1, minor: 0)
                 );
 
+            var azuriteContainer = await GetAzuriteContainer();
+
+            var configuration = CreateConfiguration();
+            configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
+
             var publicBlobStorageService = new PublicBlobStorageService(
                 logger: new Logger<BlobStorageService>(new LoggerFactory()),
-                configuration: await CreateConfigurtionWithAzurite());
+                configuration: configuration);
 
             var formFile = CreateDataCsvFormFile(""""
                                                  column_1,column_2,column_3
@@ -1984,9 +1989,14 @@ public async Task FetchCsvWithSingleDataRow_Success()
                 .WithFile(_fixture.DefaultFile()
                     .WithDataSetFileMeta(_fixture.DefaultDataSetFileMeta()));
 
+            var azuriteContainer = await GetAzuriteContainer();
+
+            var configuration = CreateConfiguration();
+            configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
+
             var publicBlobStorageService = new PublicBlobStorageService(
                 logger: new Logger<BlobStorageService>(new LoggerFactory()),
-                configuration: await CreateConfigurtionWithAzurite());
+                configuration: configuration);
 
             var formFile = CreateDataCsvFormFile("""
                                                  column_1,column_2,column_3
@@ -2052,9 +2062,14 @@ public async Task FetchDataSetFiltersOrdered_Success()
                             new FilterMeta { Id = filter2Id, Label = "Filter 2", ColumnName = "filter_2", },
                         ])));
 
+            var azuriteContainer = await GetAzuriteContainer();
+
+            var configuration = CreateConfiguration();
+            configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
+
             var publicBlobStorageService = new PublicBlobStorageService(
                 logger: new Logger<BlobStorageService>(new LoggerFactory()),
-                configuration: await CreateConfigurtionWithAzurite());
+                configuration: configuration);
 
             var formFile = CreateDataCsvFormFile("""
                                                  column_1
@@ -2119,9 +2134,14 @@ public async Task FetchDataSetIndicatorsOrdered_Success()
                             new IndicatorMeta { Id = indicator4Id, Label = "Indicator 4", ColumnName = "indicator_4", },
                         ])));
 
+            var azuriteContainer = await GetAzuriteContainer();
+
+            var configuration = CreateConfiguration();
+            configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
+
             var publicBlobStorageService = new PublicBlobStorageService(
                 logger: new Logger<BlobStorageService>(new LoggerFactory()),
-                configuration: await CreateConfigurtionWithAzurite());
+                configuration: configuration);
 
             var formFile = CreateDataCsvFormFile("""
                                                  column_1
@@ -2179,9 +2199,14 @@ public async Task FetchVariables_Success()
                             new IndicatorMeta { Id = Guid.NewGuid(), Label = "Indicator 4", ColumnName = "F_indicator_4", },
                         ])));
 
+            var azuriteContainer = await GetAzuriteContainer();
+
+            var configuration = CreateConfiguration();
+            configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
+
             var publicBlobStorageService = new PublicBlobStorageService(
                 logger: new Logger<BlobStorageService>(new LoggerFactory()),
-                configuration: await CreateConfigurtionWithAzurite());
+                configuration: configuration);
 
             var formFile = CreateDataCsvFormFile("""
                                                  column_1
@@ -2259,9 +2284,14 @@ public async Task FetchDataSetFootnotes_Success()
                     .WithFilters(new List<Filter> { filter }))
                 .Generate();
 
-        var publicBlobStorageService = new PublicBlobStorageService(
+            var azuriteContainer = await GetAzuriteContainer();
+
+            var configuration = CreateConfiguration();
+            configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
+
+            var publicBlobStorageService = new PublicBlobStorageService(
                 logger: new Logger<BlobStorageService>(new LoggerFactory()),
-                configuration: await CreateConfigurtionWithAzurite());
+                configuration: configuration);
 
             var formFile = CreateDataCsvFormFile("""
                                                  column_1
@@ -2378,9 +2408,14 @@ public async Task AmendmentNotPublished_ReturnsOk()
                 .WithReleaseVersion(publication.ReleaseVersions[2]) // the draft version
                 .WithFile(file);
 
+            var azuriteContainer = await GetAzuriteContainer();
+
+            var configuration = CreateConfiguration();
+            configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
+
             var publicBlobStorageService = new PublicBlobStorageService(
                 logger: new Logger<BlobStorageService>(new LoggerFactory()),
-                configuration: await CreateConfigurtionWithAzurite());
+                configuration: configuration);
 
             var formFile = CreateDataCsvFormFile("""
                                                  column_1
@@ -2476,20 +2511,23 @@ private WebApplicationFactory<TestStartup> BuildApp(
             });
     }
 
-    private async Task<IConfiguration> CreateConfigurtionWithAzurite()
+    private async Task<AzuriteContainer> GetAzuriteContainer()
     {
         var azuriteContainer = new AzuriteBuilder()
             .WithImage("mcr.microsoft.com/azure-storage/azurite:3.27.0")
             .WithHostname("data-storage-test")
             .Build();
         await azuriteContainer.StartAsync();
+        return azuriteContainer;
+    }
 
+    private IConfiguration CreateConfiguration()
+    {
         var configuration = new ConfigurationBuilder()
             .SetBasePath(Directory.GetCurrentDirectory())
             .AddJsonFile("appsettings.IntegrationTest.json", optional: false)
             .AddEnvironmentVariables()
             .Build();
-        configuration["PublicStorage"] = azuriteContainer.GetConnectionString();
 
         return configuration;
     }

src/GovUk.Education.ExploreEducationStatistics.Content.Model/DataSetFileMeta.cs

@@ -18,7 +18,7 @@ public class DataSetFileMeta
     public TimeIdentifier? TimeIdentifier { get; set; } // EES-4918 to remove
 
     [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
-    public List<int>? Years { get; set; } = new(); // EES-4918 to remove
+    public List<int>? Years { get; set; } // EES-4918 to remove
 
     public required TimePeriodRangeMeta TimePeriodRange { get; set; }