-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into remote-recovery-it-fix
Signed-off-by: Varun Bansal <bansvaru@amazon.com>
- Loading branch information
Showing
237 changed files
with
8,527 additions
and
1,330 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
81 changes: 81 additions & 0 deletions
81
.../main/java/org/opensearch/benchmark/index/mapper/CustomBinaryDocValuesFieldBenchmark.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.benchmark.index.mapper; | ||
|
|
||
| import org.apache.lucene.util.BytesRef; | ||
| import org.opensearch.index.mapper.BinaryFieldMapper; | ||
| import org.openjdk.jmh.annotations.Benchmark; | ||
| import org.openjdk.jmh.annotations.BenchmarkMode; | ||
| import org.openjdk.jmh.annotations.Fork; | ||
| import org.openjdk.jmh.annotations.Measurement; | ||
| import org.openjdk.jmh.annotations.Mode; | ||
| import org.openjdk.jmh.annotations.OutputTimeUnit; | ||
| import org.openjdk.jmh.annotations.Param; | ||
| import org.openjdk.jmh.annotations.Scope; | ||
| import org.openjdk.jmh.annotations.Setup; | ||
| import org.openjdk.jmh.annotations.State; | ||
| import org.openjdk.jmh.annotations.Warmup; | ||
| import org.openjdk.jmh.infra.Blackhole; | ||
|
|
||
| import java.util.concurrent.ThreadLocalRandom; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| @Warmup(iterations = 1) | ||
| @Measurement(iterations = 1) | ||
| @Fork(1) | ||
| @BenchmarkMode(Mode.Throughput) | ||
| @OutputTimeUnit(TimeUnit.MILLISECONDS) | ||
| @State(Scope.Thread) | ||
| @SuppressWarnings("unused") // invoked by benchmarking framework | ||
| public class CustomBinaryDocValuesFieldBenchmark { | ||
|
|
||
| static final String FIELD_NAME = "dummy"; | ||
| static final String SEED_VALUE = "seed"; | ||
|
|
||
| @Benchmark | ||
| public void add(CustomBinaryDocValuesFieldBenchmark.BenchmarkParameters parameters, Blackhole blackhole) { | ||
| // Don't use the parameter binary doc values object. | ||
| // Start with a fresh object every call and add maximum number of entries | ||
| BinaryFieldMapper.CustomBinaryDocValuesField customBinaryDocValuesField = new BinaryFieldMapper.CustomBinaryDocValuesField( | ||
| FIELD_NAME, | ||
| new BytesRef(SEED_VALUE).bytes | ||
| ); | ||
| for (int i = 0; i < parameters.maximumNumberOfEntries; ++i) { | ||
| ThreadLocalRandom.current().nextBytes(parameters.bytes); | ||
| customBinaryDocValuesField.add(parameters.bytes); | ||
| } | ||
| } | ||
|
|
||
| @Benchmark | ||
| public void binaryValue(CustomBinaryDocValuesFieldBenchmark.BenchmarkParameters parameters, Blackhole blackhole) { | ||
| blackhole.consume(parameters.customBinaryDocValuesField.binaryValue()); | ||
| } | ||
|
|
||
| @State(Scope.Benchmark) | ||
| public static class BenchmarkParameters { | ||
| @Param({ "8", "32", "128", "512" }) | ||
| int maximumNumberOfEntries; | ||
|
|
||
| @Param({ "8", "32", "128", "512" }) | ||
| int entrySize; | ||
|
|
||
| BinaryFieldMapper.CustomBinaryDocValuesField customBinaryDocValuesField; | ||
| byte[] bytes; | ||
|
|
||
| @Setup | ||
| public void setup() { | ||
| customBinaryDocValuesField = new BinaryFieldMapper.CustomBinaryDocValuesField(FIELD_NAME, new BytesRef(SEED_VALUE).bytes); | ||
| bytes = new byte[entrySize]; | ||
| for (int i = 0; i < maximumNumberOfEntries; ++i) { | ||
| ThreadLocalRandom.current().nextBytes(bytes); | ||
| customBinaryDocValuesField.add(bytes); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
114 changes: 114 additions & 0 deletions
114
libs/common/src/main/java/org/opensearch/common/crypto/CryptoHandler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.common.crypto; | ||
|
|
||
| import org.opensearch.common.io.InputStreamContainer; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
|
|
||
| /** | ||
| * Crypto provider abstractions for encryption and decryption of data. Allows registering multiple providers | ||
| * for defining different ways of encrypting or decrypting data. | ||
| * | ||
| * T - Encryption Metadata / CryptoContext | ||
| * U - Parsed Encryption Metadata / CryptoContext | ||
| */ | ||
| public interface CryptoHandler<T, U> { | ||
|
|
||
| /** | ||
| * To initialise or create a new crypto metadata to be used in encryption. This is needed to set the context before | ||
| * beginning encryption. | ||
| * | ||
| * @return crypto metadata instance | ||
| */ | ||
| T initEncryptionMetadata(); | ||
|
|
||
| /** | ||
| * To load crypto metadata to be used in encryption from content header. | ||
| * Note that underlying information in the loaded metadata object is same as present in the object created during | ||
| * encryption but object type may differ. | ||
| * | ||
| * @return crypto metadata instance used in decryption. | ||
| */ | ||
| U loadEncryptionMetadata(EncryptedHeaderContentSupplier encryptedHeaderContentSupplier) throws IOException; | ||
|
|
||
| /** | ||
| * Few encryption algorithms have certain conditions on the unit of content to be encrypted. This requires the | ||
| * content size to be re adjusted in order to fulfil these conditions for partial writes. If write requests for | ||
| * encryption of a part of content do not fulfil these conditions then encryption fails or can result in corrupted | ||
| * content depending on the algorithm used. This method exposes a means to re-adjust sizes of such writes. | ||
| * | ||
| * @param cryptoContext crypto metadata instance | ||
| * @param contentSize Size of the raw content | ||
| * @return Adjusted size of the content. | ||
| */ | ||
| long adjustContentSizeForPartialEncryption(T cryptoContext, long contentSize); | ||
|
|
||
| /** | ||
| * Estimate length of the encrypted content. It should only be used to determine length of entire content after | ||
| * encryption. | ||
| * | ||
| * @param cryptoContext crypto metadata instance consisting of encryption metadata used in encryption. | ||
| * @param contentLength Size of the raw content | ||
| * @return Calculated size of the encrypted content. | ||
| */ | ||
| long estimateEncryptedLengthOfEntireContent(T cryptoContext, long contentLength); | ||
|
|
||
| /** | ||
| * For given encrypted content length, estimate the length of the decrypted content. | ||
| * @param cryptoContext crypto metadata instance consisting of encryption metadata used in encryption. | ||
| * @param contentLength Size of the encrypted content | ||
| * @return Calculated size of the decrypted content. | ||
| */ | ||
| long estimateDecryptedLength(U cryptoContext, long contentLength); | ||
|
|
||
| /** | ||
| * Wraps a raw InputStream with encrypting stream | ||
| * | ||
| * @param encryptionMetadata created earlier to set the crypto metadata. | ||
| * @param stream Raw InputStream to encrypt | ||
| * @return encrypting stream wrapped around raw InputStream. | ||
| */ | ||
| InputStreamContainer createEncryptingStream(T encryptionMetadata, InputStreamContainer stream); | ||
|
|
||
| /** | ||
| * Provides encrypted stream for a raw stream emitted for a part of content. | ||
| * | ||
| * @param cryptoContext crypto metadata instance. | ||
| * @param stream raw stream for which encrypted stream has to be created. | ||
| * @param totalStreams Number of streams being used for the entire content. | ||
| * @param streamIdx Index of the current stream. | ||
| * @return Encrypted stream for the provided raw stream. | ||
| */ | ||
| InputStreamContainer createEncryptingStreamOfPart(T cryptoContext, InputStreamContainer stream, int totalStreams, int streamIdx); | ||
|
|
||
| /** | ||
| * This method accepts an encrypted stream and provides a decrypting wrapper. | ||
| * @param encryptingStream to be decrypted. | ||
| * @return Decrypting wrapper stream | ||
| */ | ||
| InputStream createDecryptingStream(InputStream encryptingStream); | ||
|
|
||
| /** | ||
| * This method creates a {@link DecryptedRangedStreamProvider} which provides a wrapped stream to decrypt the | ||
| * underlying stream. This also provides adjusted range against the actual range which should be used for fetching | ||
| * and supplying the encrypted content for decryption. Extra content outside the range is trimmed down and returned | ||
| * by the decrypted stream. | ||
| * For partial reads of encrypted content, few algorithms require the range of content to be adjusted for | ||
| * successful decryption. Adjusted range may or may not be same as the provided range. If range is adjusted then | ||
| * starting offset of resultant range can be lesser than the starting offset of provided range and end | ||
| * offset can be greater than the ending offset of the provided range. | ||
| * | ||
| * @param cryptoContext crypto metadata instance. | ||
| * @param startPosOfRawContent starting position in the raw/decrypted content | ||
| * @param endPosOfRawContent ending position in the raw/decrypted content | ||
| */ | ||
| DecryptedRangedStreamProvider createDecryptingStreamOfRange(U cryptoContext, long startPosOfRawContent, long endPosOfRawContent); | ||
| } |
42 changes: 42 additions & 0 deletions
42
libs/common/src/main/java/org/opensearch/common/crypto/DataKeyPair.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
| package org.opensearch.common.crypto; | ||
|
|
||
| /** | ||
| * Key pair generated by {@link MasterKeyProvider} | ||
| */ | ||
| public class DataKeyPair { | ||
| private final byte[] rawKey; | ||
| private final byte[] encryptedKey; | ||
|
|
||
| /** | ||
| * Constructor to initialize key-pair values | ||
| * @param rawKey Unencrypted data key used for encryption and decryption | ||
| * @param encryptedKey Encrypted version of rawKey | ||
| */ | ||
| public DataKeyPair(byte[] rawKey, byte[] encryptedKey) { | ||
| this.rawKey = rawKey; | ||
| this.encryptedKey = encryptedKey; | ||
| } | ||
|
|
||
| /** | ||
| * Returns raw key | ||
| * @return raw/decrypted key | ||
| */ | ||
| public byte[] getRawKey() { | ||
| return rawKey; | ||
| } | ||
|
|
||
| /** | ||
| * Returns encrypted key | ||
| * @return encrypted key | ||
| */ | ||
| public byte[] getEncryptedKey() { | ||
| return encryptedKey; | ||
| } | ||
| } |
49 changes: 49 additions & 0 deletions
49
libs/common/src/main/java/org/opensearch/common/crypto/DecryptedRangedStreamProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.common.crypto; | ||
|
|
||
| import java.io.InputStream; | ||
| import java.util.function.UnaryOperator; | ||
|
|
||
| /** | ||
| * Contains adjusted range of partial encrypted content which needs to be used for decryption. | ||
| */ | ||
| public class DecryptedRangedStreamProvider { | ||
|
|
||
| private final long[] adjustedRange; | ||
| private final UnaryOperator<InputStream> decryptedStreamProvider; | ||
|
|
||
| /** | ||
| * To construct adjusted encrypted range. | ||
| * @param adjustedRange range of partial encrypted content which needs to be used for decryption. | ||
| * @param decryptedStreamProvider stream provider for decryption and range re-adjustment. | ||
| */ | ||
| public DecryptedRangedStreamProvider(long[] adjustedRange, UnaryOperator<InputStream> decryptedStreamProvider) { | ||
| this.adjustedRange = adjustedRange; | ||
| this.decryptedStreamProvider = decryptedStreamProvider; | ||
| } | ||
|
|
||
| /** | ||
| * Adjusted range of partial encrypted content which needs to be used for decryption. | ||
| * @return adjusted range | ||
| */ | ||
| public long[] getAdjustedRange() { | ||
| return adjustedRange; | ||
| } | ||
|
|
||
| /** | ||
| * A utility stream provider which supplies the stream responsible for decrypting the content and reading the | ||
| * desired range of decrypted content by skipping extra content which got decrypted as a result of range adjustment. | ||
| * @return stream provider for decryption and supplying the desired range of content. | ||
| */ | ||
| public UnaryOperator<InputStream> getDecryptedStreamProvider() { | ||
| return decryptedStreamProvider; | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.