From 6bb738195fdfd8d3fcd4003721c1ffe3f6a920a0 Mon Sep 17 00:00:00 2001
From: Andrew Gaul <andrew@gaul.org>
Date: Wed, 21 Feb 2024 21:57:51 +0900
Subject: [PATCH] Add a test for uploading large objects

This is disabled by default.  Also rework upload helper to avoid using
an unbounded amount of memory.
---
 .../internal/BaseBlobIntegrationTest.java     | 33 +++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
index d885f4530e..b681f0d08d 100644
--- a/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
+++ b/blobstore/src/test/java/org/jclouds/blobstore/integration/internal/BaseBlobIntegrationTest.java
@@ -699,6 +699,22 @@ public void testPutMultipartByteSource() throws Exception {
       testPut(payload, hashCode, payload, length, new PutOptions().multipart(true));
    }
 
+   @Test(groups = { "integration", "live" }, enabled = false)
+   public void testPutMultipartByteSourceBig() throws Exception {
+      long length = 5L * 1024 * 1024 * 1024 + 1;
+      BlobStore blobStore = view.getBlobStore();
+      MultipartUploadSlicingAlgorithm algorithm = new MultipartUploadSlicingAlgorithm(
+              blobStore.getMinimumMultipartPartSize(), blobStore.getMaximumMultipartPartSize(),
+              blobStore.getMaximumNumberOfParts());
+      // make sure that we are creating multiple parts
+      assertThat(algorithm.calculateChunkSize(length)).isLessThan(length);
+      // TODO: this is inefficient due to skip recomputing entire stream
+      ByteSource byteSource = TestUtils.randomByteSource().slice(0, length);
+      Payload payload = new ByteSourcePayload(byteSource);
+      HashCode hashCode = byteSource.hash(Hashing.md5());
+      testPut(payload, hashCode, payload, length, new PutOptions().multipart(true));
+   }
+
    @Test(groups = { "integration", "live" })
    public void testPutMultipartInputStream() throws Exception {
       long length = Math.max(getMinimumMultipartBlobSize(), MultipartUploadSlicingAlgorithm.DEFAULT_PART_SIZE + 1);
@@ -858,12 +874,17 @@ private void testPut(Payload payload, HashCode hashCode, Payload expectedPayload
          Blob blob = blobStore.getBlob(container, blobName);
          assertThat(blob.getMetadata().getContentMetadata().getContentLength()).isEqualTo(length);
 
-         InputStream is = null;
-         try {
-            is = blob.getPayload().openStream();
-            assertThat(is).hasContentEqualTo(expectedPayload.openStream());
-         } finally {
-            Closeables2.closeQuietly(is);
+         try (InputStream actual = blob.getPayload().openStream();
+              InputStream expected = expectedPayload.openStream()) {
+            while (length > 0) {
+               int bufferLength = (int) Math.min(1 << 20, length);
+               byte[] actualBuffer = new byte[bufferLength];
+               ByteStreams.readFully(actual, actualBuffer);
+               byte[] expectedBuffer = new byte[bufferLength];
+               ByteStreams.readFully(expected, expectedBuffer);
+               assertThat(actualBuffer).isEqualTo(expectedBuffer);
+               length -= bufferLength;
+            }
          }
          validateMetadata(blob.getMetadata(), container, blob.getMetadata().getName());
          checkContentMetadata(blob);