Reduce memory copy in zstd compression (#7681)

* 1. reduce memory copy in zstd compression Signed-off-by: luyuncheng <luyuncheng@bytedance.com> * Add Change log Signed-off-by: luyuncheng <luyuncheng@bytedance.com> --------- Signed-off-by: luyuncheng <luyuncheng@bytedance.com>
opensearch-project · May 31, 2023 · a0a8a18 · a0a8a18
1 parent a201400
commit a0a8a18
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
+- Reduce memory copy in zstd compression ([#7681](https://github.com/opensearch-project/OpenSearch/pull/7681))
 
 ### Deprecated
 

diff --git a/...tom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/...tom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java
@@ -77,7 +77,7 @@ private void doCompress(byte[] bytes, int offset, int length, ZstdCompressCtx cc
                 return;
             }
             final int maxCompressedLength = (int) Zstd.compressBound(length);
-            compressedBuffer = ArrayUtil.grow(compressedBuffer, maxCompressedLength);
+            compressedBuffer = ArrayUtil.growNoCopy(compressedBuffer, maxCompressedLength);
 
             int compressedSize = cctx.compressByteArray(compressedBuffer, 0, compressedBuffer.length, bytes, offset, length);
 
@@ -139,7 +139,7 @@ private void doDecompress(DataInput in, ZstdDecompressCtx dctx, BytesRef bytes,
                 return;
             }
 
-            compressedBuffer = ArrayUtil.grow(compressedBuffer, compressedLength);
+            compressedBuffer = ArrayUtil.growNoCopy(compressedBuffer, compressedLength);
             in.readBytes(compressedBuffer, 0, compressedLength);
 
             bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + decompressedLen);
@@ -161,7 +161,7 @@ public void decompress(DataInput in, int originalLength, int offset, int length,
             }
             final int dictLength = in.readVInt();
             final int blockLength = in.readVInt();
-            bytes.bytes = ArrayUtil.grow(bytes.bytes, dictLength);
+            bytes.bytes = ArrayUtil.growNoCopy(bytes.bytes, dictLength);
             bytes.offset = bytes.length = 0;
 
             try (ZstdDecompressCtx dctx = new ZstdDecompressCtx()) {

diff --git a/...decs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/...decs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java
@@ -83,7 +83,7 @@ private void compress(byte[] bytes, int offset, int length, DataOutput out) thro
                 }
 
                 final int maxCompressedLength = (int) Zstd.compressBound(l);
-                compressedBuffer = ArrayUtil.grow(compressedBuffer, maxCompressedLength);
+                compressedBuffer = ArrayUtil.growNoCopy(compressedBuffer, maxCompressedLength);
 
                 int compressedSize = (int) Zstd.compressByteArray(
                     compressedBuffer,
@@ -151,7 +151,7 @@ public void decompress(DataInput in, int originalLength, int offset, int length,
                 if (compressedLength == 0) {
                     return;
                 }
-                compressed = ArrayUtil.grow(compressed, compressedLength);
+                compressed = ArrayUtil.growNoCopy(compressed, compressedLength);
                 in.readBytes(compressed, 0, compressedLength);
 
                 int l = Math.min(blockLength, originalLength - offsetInBlock);