Feat/checksum (#186)

* implement checksum verification

* seperate locationUrl retrieval from upload method

* refactor getbloburi and gethash into UploadAttachment

* format document

* remove unused methods

* Check if rows updated are 1, otherwise return false

* Create static class containing text used in AttachmentStatus to provide more information

* Add checksum to factories

* fix: check if checksum is empty before comparison

* fix missing data in attachment overview

* add missing expiration date

* add missing data

* remove duplicate from other pr

* First tests written

* cleanup all attachmentData

* fix attachmentFactory in test

* fix ExternalReferences

* Add checksum to GetAttachmentOverviewResponse and AttachmentOverviewMapper

* support null values for list in correspondence requests

* send correspondenceAttachment id on get operations for attachment

* More tests

* Implement changes to UploadHelper

* fix: use AttachmentId instead of AttachmentEntity

* fix possible memory leak

* add more errors

* add data to error message for testing purposes

* compare expectedChanges with actual rows updated for debugging

* update current testing process

* add fix for multiple correspondences

* cleanup

* fix create correspondence operations

* change to upload faile

* more cleanup

* fix wrong bool check for rows updated

---------

Co-authored-by: Hammerbeck <andreas.hammerbeck@digdir.no>
Co-authored-by: Andreas Hammerbeck <Andreas_93@hotmail.com>

Test/Altinn.Correspondence.Tests/AttachmentControllerTests.cs

@@ -1,9 +1,11 @@
 using Altinn.Correspondece.Tests.Factories;
 using Altinn.Correspondence.API.Models;
 using Altinn.Correspondence.API.Models.Enums;
+using Altinn.Correspondence.Tests.Helpers;
 using Microsoft.AspNetCore.Mvc;
 using System.Net;
 using System.Net.Http.Json;
+using System.Text;
 using System.Text.Json;
 
 namespace Altinn.Correspondence.Tests;
@@ -149,6 +151,99 @@ public async Task UploadAttachmentData_Succeeds_DownloadedBytesAreSame()
         // Assert that the uploaded and downloaded bytes are the same
         Assert.Equal(originalAttachmentData, downloadedAttachmentData);
     }
+    [Fact]
+    public async Task UploadAtttachmentData_ChecksumCorrect_Succeeds()
+    {
+        // Arrange
+        var attachment = InitializeAttachmentFactory.BasicAttachment();
+        var data = "This is the contents of the uploaded file";
+        var byteData = Encoding.UTF8.GetBytes(data);
+        var checksum = Utils.CalculateChecksum(byteData);
+        attachment.Checksum = checksum;
+
+        // Act
+        var initializeResponse = await _client.PostAsJsonAsync("correspondence/api/v1/attachment", attachment);
+        initializeResponse.EnsureSuccessStatusCode();
+        var attachmentId = await initializeResponse.Content.ReadAsStringAsync();
+        var content = new ByteArrayContent(byteData);
+        var uploadResponse = await UploadAttachment(attachmentId, content);
+
+        // Assert
+        Assert.True(uploadResponse.IsSuccessStatusCode, await uploadResponse.Content.ReadAsStringAsync());
+    }
+    [Fact]
+    public async Task UploadAttachment_MismatchChecksum_Fails()
+    {
+        // Arrange
+        var attachment = InitializeAttachmentFactory.BasicAttachment();
+        var data = "This is the contents of the uploaded file";
+        var byteData = Encoding.UTF8.GetBytes(data);
+        var checksum = Utils.CalculateChecksum(byteData);
+        attachment.Checksum = checksum;
+
+        var modifiedByteData = Encoding.UTF8.GetBytes("This is NOT the contents of the uploaded file");
+        var modifiedContent = new ByteArrayContent(modifiedByteData);
+
+        // Act
+        var initializeResponse = await _client.PostAsJsonAsync("correspondence/api/v1/attachment", attachment);
+        initializeResponse.EnsureSuccessStatusCode();
+        var attachmentId = await initializeResponse.Content.ReadAsStringAsync();
+        var uploadResponse = await UploadAttachment(attachmentId, modifiedContent);
+
+        // Assert
+        Assert.False(uploadResponse.IsSuccessStatusCode);
+        Assert.Equal(HttpStatusCode.BadRequest, uploadResponse.StatusCode);
+    }
+    [Fact]
+    public async Task UploadAttachment_NoChecksumSetWhenInitialized_ChecksumSetAfterUpload()
+    {
+        // Arrange
+        var attachment = InitializeAttachmentFactory.BasicAttachment();
+        var initializeResponse = await _client.PostAsJsonAsync("correspondence/api/v1/attachment", attachment);
+        initializeResponse.EnsureSuccessStatusCode();
+        var attachmentId = await initializeResponse.Content.ReadAsStringAsync();
+        var prevOverview = await _client.GetFromJsonAsync<AttachmentOverviewExt>($"correspondence/api/v1/attachment/{attachmentId}", _responseSerializerOptions);
+
+        var data = "This is the contents of the uploaded file";
+        var byteData = Encoding.UTF8.GetBytes(data);
+        var checksum = Utils.CalculateChecksum(byteData);
+
+        var content = new ByteArrayContent(byteData);
+
+        // Act
+        await UploadAttachment(attachmentId, content);
+        var attachmentOverview = await _client.GetFromJsonAsync<AttachmentOverviewExt>($"correspondence/api/v1/attachment/{attachmentId}", _responseSerializerOptions);
+
+        // Assert
+        Assert.Empty(prevOverview.Checksum);
+        Assert.NotEmpty(attachmentOverview.Checksum);
+        Assert.Equal(checksum, attachmentOverview.Checksum);
+    }
+    [Fact]
+    public async Task UploadAttachment_ChecksumSetWhenInitialized_SameChecksumSetAfterUpload()
+    {
+        // Arrange
+        var attachment = InitializeAttachmentFactory.BasicAttachment();
+        var data = "This is the contents of the uploaded file";
+        var byteData = Encoding.UTF8.GetBytes(data);
+        var checksum = Utils.CalculateChecksum(byteData);
+        attachment.Checksum = checksum;
+
+        var content = new ByteArrayContent(byteData);
+
+        // Act
+        var initializeResponse = await _client.PostAsJsonAsync("correspondence/api/v1/attachment", attachment);
+        initializeResponse.EnsureSuccessStatusCode();
+        var attachmentId = await initializeResponse.Content.ReadAsStringAsync();
+        var prevOverview = await _client.GetFromJsonAsync<AttachmentOverviewExt>($"correspondence/api/v1/attachment/{attachmentId}", _responseSerializerOptions);
+        await UploadAttachment(attachmentId, content);
+        var attachmentOverview = await _client.GetFromJsonAsync<AttachmentOverviewExt>($"correspondence/api/v1/attachment/{attachmentId}", _responseSerializerOptions);
+
+        // Assert
+        Assert.NotEmpty(prevOverview.Checksum);
+        Assert.NotEmpty(attachmentOverview.Checksum);
+        Assert.Equal(prevOverview.Checksum, attachmentOverview.Checksum);
+    }
 
     [Fact]
     public async Task DeleteAttachment_WhenAttachmentDoesNotExist_ReturnsNotFound()

Test/Altinn.Correspondence.Tests/CorrespondenceControllerTests.cs

@@ -235,6 +235,7 @@ public async Task UploadMultipleCorrespondence_With_Multiple_Files()
                 FileName = file2.FileName,
                 IsEncrypted = false,
             }};
+
         var formData = CorrespondenceToFormData(correspondence, "Correspondence.");
 
         formData.Add(new StreamContent(fileStream), "attachments", file.FileName);

Test/Altinn.Correspondence.Tests/Factories/InitializeAttachmentFactory.cs

@@ -1,5 +1,6 @@
+using System.Text;
 using Altinn.Correspondence.API.Models;
-using Altinn.Correspondence.API.Models.Enums;
+using Altinn.Correspondence.Tests.Helpers;
 
 namespace Altinn.Correspondece.Tests.Factories;
 internal static class InitializeAttachmentFactory
@@ -15,4 +16,4 @@ internal static class InitializeAttachmentFactory
         FileName = "test-file",
         IsEncrypted = false
     };
-}
+}

Test/Altinn.Correspondence.Tests/Factories/InitializeCorrespondenceFactory.cs

@@ -1,5 +1,6 @@
 using Altinn.Correspondence.API.Models;
 using Altinn.Correspondence.API.Models.Enums;
+using Altinn.Correspondence.Tests.Helpers;
 
 namespace Altinn.Correspondece.Tests.Factories;
 internal static class InitializeCorrespondenceFactory
@@ -24,7 +25,7 @@ internal static class InitializeCorrespondenceFactory
                     RestrictionName = "testFile2",
                     SendersReference = "1234",
                     FileName = "test-fil2e",
-                    IsEncrypted = false,
+                    IsEncrypted = false
                 }
             },
         },

Test/Altinn.Correspondence.Tests/Helpers/Utils.cs

@@ -0,0 +1,14 @@
+namespace Altinn.Correspondence.Tests.Helpers
+{
+    internal static class Utils
+    {
+        public static string CalculateChecksum(byte[] data)
+        {
+            using (var md5 = System.Security.Cryptography.MD5.Create())
+            {
+                byte[] hash = md5.ComputeHash(data);
+                return BitConverter.ToString(hash).Replace("-", "").ToLower();
+            }
+        }
+    }
+}

src/Altinn.Correspondence.API/Mappers/AttachmentOverviewMapper.cs

@@ -18,11 +18,12 @@ internal static AttachmentOverviewExt MapToExternal(GetAttachmentOverviewRespons
             RestrictionName = attachmentOverview.RestrictionName,
             Status = (AttachmentStatusExt)attachmentOverview.Status,
             StatusText = attachmentOverview.StatusText,
+            Checksum = attachmentOverview.Checksum,
             DataLocationUrl = attachmentOverview.DataLocationUrl,
             StatusChanged = attachmentOverview.StatusChanged,
             DataType = attachmentOverview.DataType,
             SendersReference = attachmentOverview.SendersReference,
-            CorrespondenceIds = attachmentOverview.CorrespondenceIds
+            CorrespondenceIds = attachmentOverview.CorrespondenceIds ?? new List<Guid>(),
         };
         return attachment;
     }

src/Altinn.Correspondence.Application/AttachmentStatusText.cs

@@ -0,0 +1,9 @@
+namespace Altinn.Correspondence.Application
+{
+    public static class AttachmentStatusText
+    {
+        public const string InvalidLocationUrl = "Could not get data location url";
+        public const string ChecksumMismatch = "Checksum mismatch";
+        public const string UploadFailed = "Upload failed";
+    }
+}

src/Altinn.Correspondence.Application/Errors.cs

@@ -28,4 +28,6 @@ public static class Errors
     public static Error UploadedFilesDoesNotMatchAttachments = new Error(20, "Mismatch between uploaded files and attachment metadata", HttpStatusCode.BadRequest);
     public static Error DuplicateRecipients = new Error(21, "Recipients must be unique", HttpStatusCode.BadRequest);
     public static Error MultipleCorrespondenceNoAttachments = new Error(22, "When uploading multiple correspondences, either upload or use existing attachments", HttpStatusCode.BadRequest);
+    public static Error HashError = new Error(23, "Checksum mismatch", HttpStatusCode.BadRequest);
+    public static Error DataLocationNotFound = new Error(24, "Could not get data location url", HttpStatusCode.BadRequest);
 }

src/Altinn.Correspondence.Application/GetAttachmentOverview/GetAttachmentOverviewHandler.cs

@@ -40,6 +40,7 @@ public async Task<OneOf<GetAttachmentOverviewResponse, Error>> Process(Guid atta
             ResourceId = attachment.ResourceId,
             DataLocationUrl = attachment.DataLocationUrl,
             Name = attachment.FileName,
+            Checksum = attachment.Checksum,
             Status = attachmentStatus.Status,
             StatusText = attachmentStatus.StatusText,
             StatusChanged = attachmentStatus.StatusChanged,

src/Altinn.Correspondence.Application/GetAttachmentOverview/GetAttachmentOverviewResponse.cs

@@ -20,6 +20,8 @@ public class GetAttachmentOverviewResponse
     public string FileName { get; set; } = string.Empty;
 
     public string? Name { get; set; } = string.Empty;
+
+    public string? Checksum { get; set; } = string.Empty;
 
     public string Sender { get; set; } = string.Empty;
 

src/Altinn.Correspondence.Application/Helpers/InitializeCorrespondenceHelper.cs

@@ -91,17 +91,22 @@ public CorrespondenceStatus GetInitializeCorrespondenceStatus(CorrespondenceEnti
             return status;
         }
 
-        public async Task<Error?> UploadAttachments(CorrespondenceEntity correspondence, List<IFormFile> attachments, CancellationToken cancellationToken)
+        public async Task<Error?> UploadAttachments(List<AttachmentEntity> correspondenceAttachments, List<IFormFile> files, CancellationToken cancellationToken)
         {
             UploadHelper uploadHelper = new UploadHelper(_correspondenceRepository, _correspondenceStatusRepository, _attachmentStatusRepository, _attachmentRepository, _storageRepository, _hostEnvironment);
-            foreach (var file in attachments)
+            foreach (var file in files)
             {
-                var attachment = correspondence.Content?.Attachments.FirstOrDefault(a => a.Attachment.FileName.ToLower() == file.FileName.ToLower());
-                if (attachment == null || attachment.Attachment == null)
+                var attachment = correspondenceAttachments.FirstOrDefault(a => a.FileName.ToLower() == file.FileName.ToLower());
+
+                if (attachment == null)
                 {
                     return Errors.UploadedFilesDoesNotMatchAttachments;
                 }
-                var uploadResponse = await uploadHelper.UploadAttachment(file.OpenReadStream(), attachment.AttachmentId, cancellationToken);
+                OneOf<UploadAttachmentResponse, Error> uploadResponse;
+                await using (var f = file.OpenReadStream())
+                {
+                    uploadResponse = await uploadHelper.UploadAttachment(f, attachment.Id, cancellationToken);
+                }
                 var error = uploadResponse.Match(
                     _ => { return null; },
                     error => { return error; }
@@ -111,7 +116,7 @@ public CorrespondenceStatus GetInitializeCorrespondenceStatus(CorrespondenceEnti
             return null;
         }
 
-        public async Task<AttachmentEntity> ProcessAttachment(CorrespondenceAttachmentEntity correspondenceAttachment, CorrespondenceEntity correspondence, CancellationToken cancellationToken)
+        public async Task<AttachmentEntity> ProcessAttachment(CorrespondenceAttachmentEntity correspondenceAttachment, bool shouldSave, CancellationToken cancellationToken)
         {
             if (!String.IsNullOrEmpty(correspondenceAttachment.Attachment?.DataLocationUrl))
             {
@@ -131,7 +136,7 @@ public async Task<AttachmentEntity> ProcessAttachment(CorrespondenceAttachmentEn
             };
             var attachment = correspondenceAttachment.Attachment!;
             attachment.Statuses = status;
-            return attachment;
+            return await _attachmentRepository.InitializeAttachment(attachment, cancellationToken);
         }
     }
 }

src/Altinn.Correspondence.Application/Helpers/UploadHelper.cs

@@ -1,8 +1,10 @@
 using System;
 using Altinn.Correspondence.Application.UploadAttachment;
+using Altinn.Correspondence.Core.Exceptions;
 using Altinn.Correspondence.Core.Models;
 using Altinn.Correspondence.Core.Models.Enums;
 using Altinn.Correspondence.Core.Repositories;
+using Azure;
 using Microsoft.Extensions.Hosting;
 using OneOf;
 
@@ -32,41 +34,48 @@ public async Task<OneOf<UploadAttachmentResponse, Error>> UploadAttachment(Strea
             var attachment = await _attachmentRepository.GetAttachmentById(attachmentId, true, cancellationToken);
             if (attachment == null)
             {
-                return Errors.AttachmentNotFound;
             }
-            var currentStatus = new AttachmentStatusEntity
-            {
-                AttachmentId = attachmentId,
-                Status = AttachmentStatus.UploadProcessing,
-                StatusChanged = DateTimeOffset.UtcNow,
-                StatusText = AttachmentStatus.UploadProcessing.ToString()
-            };
-            await _attachmentStatusRepository.AddAttachmentStatus(currentStatus, cancellationToken); // TODO, with malware scan this should be set after upload
-            var dataLocationUrl = await _storageRepository.UploadAttachment(attachmentId, file, cancellationToken);
-            if (dataLocationUrl is null)
+
+            var currentStatus = await SetAttachmentStatus(attachmentId, AttachmentStatus.UploadProcessing, cancellationToken);
+            try
             {
-                currentStatus = new AttachmentStatusEntity
+                var (dataLocationUrl, checksum) = await _storageRepository.UploadAttachment(attachment, file, cancellationToken);
+
+                var isValidUpdate = await _attachmentRepository.SetDataLocationUrl(attachment, AttachmentDataLocationType.AltinnCorrespondenceAttachment, dataLocationUrl, cancellationToken);
+
+                if (string.IsNullOrWhiteSpace(attachment.Checksum))
+                {
+                    isValidUpdate |= await _attachmentRepository.SetChecksum(attachment, checksum, cancellationToken);
+                }
+
+                if (!isValidUpdate)
                 {
-                    AttachmentId = attachmentId,
-                    Status = AttachmentStatus.Failed,
-                    StatusChanged = DateTimeOffset.UtcNow,
-                    StatusText = AttachmentStatus.Failed.ToString()
-                };
-                await _attachmentStatusRepository.AddAttachmentStatus(currentStatus, cancellationToken);
+                    await SetAttachmentStatus(attachmentId, AttachmentStatus.Failed, cancellationToken, AttachmentStatusText.UploadFailed);
+                    await _storageRepository.PurgeAttachment(attachment.Id, cancellationToken);
+                    return Errors.UploadFailed;
+                }
+            }
+            catch (DataLocationUrlException)
+            {
+                await SetAttachmentStatus(attachmentId, AttachmentStatus.Failed, cancellationToken, AttachmentStatusText.InvalidLocationUrl);
+                return Errors.DataLocationNotFound;
+            }
+            catch (HashMismatchException)
+            {
+                await SetAttachmentStatus(attachmentId, AttachmentStatus.Failed, cancellationToken, AttachmentStatusText.ChecksumMismatch);
+                return Errors.HashError;
+            }
+            catch (RequestFailedException)
+            {
+                await SetAttachmentStatus(attachmentId, AttachmentStatus.Failed, cancellationToken, AttachmentStatusText.UploadFailed);
                 return Errors.UploadFailed;
             }
-            await _attachmentRepository.SetDataLocationUrl(attachment, AttachmentDataLocationType.AltinnCorrespondenceAttachment, dataLocationUrl, cancellationToken);
+
             if (_hostEnvironment.IsDevelopment()) // No malware scan when running locally
             {
-                currentStatus = new AttachmentStatusEntity
-                {
-                    AttachmentId = attachment.Id,
-                    Status = AttachmentStatus.Published,
-                    StatusChanged = DateTimeOffset.UtcNow,
-                    StatusText = AttachmentStatus.Published.ToString()
-                };
-                await _attachmentStatusRepository.AddAttachmentStatus(currentStatus, cancellationToken);
+                currentStatus = await SetAttachmentStatus(attachmentId, AttachmentStatus.Published, cancellationToken);
             }
+
             return new UploadAttachmentResponse()
             {
                 AttachmentId = attachment.Id,
@@ -75,6 +84,18 @@ public async Task<OneOf<UploadAttachmentResponse, Error>> UploadAttachment(Strea
                 StatusText = currentStatus.StatusText
             };
         }
+        private async Task<AttachmentStatusEntity> SetAttachmentStatus(Guid attachmentId, AttachmentStatus status, CancellationToken cancellationToken, string statusText = null)
+        {
+            var currentStatus = new AttachmentStatusEntity
+            {
+                AttachmentId = attachmentId,
+                Status = status,
+                StatusChanged = DateTimeOffset.UtcNow,
+                StatusText = statusText ?? status.ToString()
+            };
+            await _attachmentStatusRepository.AddAttachmentStatus(currentStatus, cancellationToken);
+            return currentStatus;
+        }
         public async Task CheckCorrespondenceStatusesAfterUploadAndPublish(Guid attachmentId, CancellationToken cancellationToken)
         {
             var attachment = await _attachmentRepository.GetAttachmentById(attachmentId, true, cancellationToken);