Fix copy large files (#8409)

Fix `OutOfMemoryError` (out of heap memory) when copying large files into a container via `GenericContainer#withCopyFileToContainer()`. Fixes #4203 --------- Co-authored-by: Eddú Meléndez <eddu.melendez@gmail.com>
testcontainers · Jul 4, 2024 · b4b1c20 · b4b1c20
1 parent a687e8a
commit b4b1c20
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 10 deletions.
diff --git a/core/src/main/java/org/testcontainers/containers/ContainerState.java b/core/src/main/java/org/testcontainers/containers/ContainerState.java
@@ -22,12 +22,12 @@
 import org.testcontainers.utility.MountableFile;
 import org.testcontainers.utility.ThrowingFunction;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -339,27 +339,37 @@ default void copyFileToContainer(MountableFile mountableFile, String containerPa
      * @param transferable file which is copied into the container
      * @param containerPath destination path inside the container
      */
-    @SneakyThrows(IOException.class)
+    @SneakyThrows({ IOException.class, InterruptedException.class })
     default void copyFileToContainer(Transferable transferable, String containerPath) {
         if (getContainerId() == null) {
             throw new IllegalStateException("copyFileToContainer can only be used with created / running container");
         }
 
         try (
-            ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-            TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(byteArrayOutputStream)
+            PipedOutputStream pipedOutputStream = new PipedOutputStream();
+            PipedInputStream pipedInputStream = new PipedInputStream(pipedOutputStream);
+            TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(pipedOutputStream)
         ) {
-            tarArchive.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
-            tarArchive.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
+            Thread thread = new Thread(() -> {
+                try {
+                    tarArchive.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+                    tarArchive.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
+
+                    transferable.transferTo(tarArchive, containerPath);
+                } finally {
+                    IOUtils.closeQuietly(tarArchive);
+                }
+            });
 
-            transferable.transferTo(tarArchive, containerPath);
-            tarArchive.finish();
+            thread.start();
 
             getDockerClient()
                 .copyArchiveToContainerCmd(getContainerId())
-                .withTarInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))
+                .withTarInputStream(pipedInputStream)
                 .withRemotePath("/")
                 .exec();
+
+            thread.join();
         }
     }
 

diff --git a/core/src/test/java/org/testcontainers/junit/FileOperationsTest.java b/core/src/test/java/org/testcontainers/junit/FileOperationsTest.java
@@ -3,15 +3,19 @@
 import com.github.dockerjava.api.exception.NotFoundException;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.CountingOutputStream;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.testcontainers.TestImages;
+import org.testcontainers.containers.Container;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy;
 import org.testcontainers.utility.MountableFile;
 
+import java.io.BufferedOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -40,6 +44,43 @@ public void copyFileToContainerFileTest() throws Exception {
         }
     }
 
+    @Test
+    public void copyLargeFilesToContainer() throws Exception {
+        File tempFile = temporaryFolder.newFile();
+        try (
+            GenericContainer<?> alpineCopyToContainer = new GenericContainer<>(TestImages.ALPINE_IMAGE) //
+                .withCommand("sleep", "infinity")
+        ) {
+            alpineCopyToContainer.start();
+            final long byteCount;
+            try (
+                FileOutputStream fos = new FileOutputStream(tempFile);
+                CountingOutputStream cos = new CountingOutputStream(fos);
+                BufferedOutputStream bos = new BufferedOutputStream(cos)
+            ) {
+                for (int i = 0; i < 0x4000; i++) {
+                    byte[] bytes = new byte[0xFFFF];
+                    bos.write(bytes);
+                }
+                bos.flush();
+                byteCount = cos.getByteCount();
+            }
+            final MountableFile mountableFile = MountableFile.forHostPath(tempFile.getPath());
+            final String containerPath = "/test.bin";
+            alpineCopyToContainer.copyFileToContainer(mountableFile, containerPath);
+
+            final Container.ExecResult execResult = alpineCopyToContainer.execInContainer( //
+                "stat",
+                "-c",
+                "%s",
+                containerPath
+            );
+            assertThat(execResult.getStdout()).isEqualToIgnoringNewLines(Long.toString(byteCount));
+        } finally {
+            tempFile.delete();
+        }
+    }
+
     @Test
     public void copyFileToContainerFolderTest() throws Exception {
         try (