From 88021247c3260b0d7874f68615ba5fb13c82bb9f Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Mon, 25 Mar 2024 09:23:01 -0400 Subject: [PATCH 01/11] Moved cryptodirectory plugin to sandbox Signed-off-by: Olasoji Denloye --- sandbox/plugins/cryptodirectory/build.gradle | 41 ++ .../index/store/CryptoDirectory.java | 530 ++++++++++++++++++ .../index/store/CryptoDirectoryFactory.java | 106 ++++ .../index/store/CryptoDirectoryPlugin.java | 69 +++ .../opensearch/index/store/package-info.java | 12 + .../index/store/CryptoDirectoryTests.java | 165 ++++++ .../CryptoDirectoryClientYamlTestSuiteIT.java | 26 + .../resources/rest-api-spec/test/10_basic.yml | 8 + 8 files changed, 957 insertions(+) create mode 100644 sandbox/plugins/cryptodirectory/build.gradle create mode 100644 sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java create mode 100644 sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java create mode 100644 sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java create mode 100644 sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/package-info.java create mode 100644 sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java create mode 100644 sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java create mode 100644 sandbox/plugins/cryptodirectory/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml diff --git a/sandbox/plugins/cryptodirectory/build.gradle b/sandbox/plugins/cryptodirectory/build.gradle new file mode 100644 index 0000000000000..737cdca3916c8 --- /dev/null +++ b/sandbox/plugins/cryptodirectory/build.gradle @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +apply plugin: 'opensearch.build' +apply plugin: 'opensearch.publish' + +opensearchplugin { + description 'Encrypts and decrypts index data at rest.' + classname 'org.opensearch.index.store.CryptoDirectoryPlugin' +} +//restResources { +// restApi { +// includeCore '_common', 'cluster', 'nodes', 'index', 'indices', 'get' +// } +//} diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java new file mode 100644 index 0000000000000..2189f29bf6811 --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java @@ -0,0 +1,530 @@ +/* * SPDX-License-Identifier: Apache-2.0 * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.store; + +import org.apache.lucene.store.BufferedIndexInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.store.OutputStreamIndexOutput; +import org.opensearch.common.Randomness; +import org.opensearch.common.crypto.DataKeyPair; +import org.opensearch.common.crypto.MasterKeyProvider; +import org.opensearch.common.util.io.IOUtils; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import java.io.EOFException; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.security.Provider; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Base64; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A hybrid directory implementation that encrypts files + * to be stored based on a user supplied key + * + * @opensearch.internal + */ +public final class CryptoDirectory extends NIOFSDirectory { + private Path location; + private Key dataKey; + private ConcurrentSkipListMap ivMap; + private final Provider provider; + + private final AtomicLong nextTempFileCounter = new AtomicLong(); + + CryptoDirectory(LockFactory lockFactory, Path location, Provider provider, MasterKeyProvider keyProvider) throws IOException { + super(location, lockFactory); + this.location = location; + ivMap = new ConcurrentSkipListMap<>(); + IndexInput in; + this.provider = provider; + + try { + in = super.openInput("ivMap", new IOContext()); + } catch (java.nio.file.NoSuchFileException nsfe) { + in = null; + } + if (in != null) { + Map tmp = in.readMapOfStrings(); + ivMap.putAll(tmp); + in.close(); + dataKey = new SecretKeySpec(keyProvider.decryptKey(getWrappedKey()), "AES"); + } else { + DataKeyPair dataKeyPair = keyProvider.generateDataPair(); + dataKey = new SecretKeySpec(dataKeyPair.getRawKey(), "AES"); + storeWrappedKey(dataKeyPair.getEncryptedKey()); + } + } + + private void storeWrappedKey(byte[] wrappedKey) { + try (IndexOutput out = super.createOutput("keyfile", new IOContext())) { + out.writeInt(wrappedKey.length); + out.writeBytes(wrappedKey, 0, wrappedKey.length); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private byte[] getWrappedKey() { + try (IndexInput in = super.openInput("keyfile", new IOContext())) { + int size = in.readInt(); + byte[] ret = new byte[size]; + in.readBytes(ret, 0, ret.length); + return ret; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * {@inheritDoc} + * @param source the file to be renamed + * @param dest the new file name + */ + @Override + public void rename(String source, String dest) throws IOException { + super.rename(source, dest); + if (!(source.contains("segments_") || source.endsWith(".si"))) ivMap.put( + getDirectory() + "/" + dest, + ivMap.remove(getDirectory() + "/" + source) + ); + } + + /** + * {@inheritDoc} + * @param name the name of the file to be opened for reading + * @param context the IO context + */ + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + if (name.contains("segments_") || name.endsWith(".si")) return super.openInput(name, context); + ensureOpen(); + ensureCanRead(name); + Path path = getDirectory().resolve(name); + FileChannel fc = FileChannel.open(path, StandardOpenOption.READ); + boolean success = false; + try { + Cipher cipher = CipherFactory.getCipher(provider); + String ivEntry = ivMap.get(getDirectory() + "/" + name); + if (ivEntry == null) throw new IOException("failed to open file. " + name); + byte[] iv = Base64.getDecoder().decode(ivEntry); + CipherFactory.initCipher(cipher, this, Optional.of(iv), Cipher.DECRYPT_MODE, 0); + final IndexInput indexInput; + indexInput = new CryptoBufferedIndexInput("CryptoBufferedIndexInput(path=\"" + path + "\")", fc, context, cipher, this); + success = true; + return indexInput; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(fc); + } + } + } + + /** + * {@inheritDoc} + * @param name the name of the file to be opened for writing + * @param context the IO context + */ + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + if (name.contains("segments_") || name.endsWith(".si")) return super.createOutput(name, context); + ensureOpen(); + OutputStream fos = Files.newOutputStream(directory.resolve(name), StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); + Cipher cipher = CipherFactory.getCipher(provider); + SecureRandom random = Randomness.createSecure(); + byte[] iv = new byte[CipherFactory.IV_ARRAY_LENGTH]; + random.nextBytes(iv); + if (dataKey == null) throw new RuntimeException("dataKey is null!"); + CipherFactory.initCipher(cipher, this, Optional.of(iv), Cipher.ENCRYPT_MODE, 0); + ivMap.put(getDirectory() + "/" + name, Base64.getEncoder().encodeToString(iv)); + return new CryptoIndexOutput(name, fos, cipher); + } + + /** + * {@inheritDoc} + * @param prefix the desired temporary file prefix + * @param suffix the desired temporary file suffix + * @param context the IO context + */ + @Override + public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException { + if (prefix.contains("segments_") || prefix.endsWith(".si")) return super.createTempOutput(prefix, suffix, context); + ensureOpen(); + String name; + while (true) { + name = getTempFileName(prefix, suffix, nextTempFileCounter.getAndIncrement()); + OutputStream fos = Files.newOutputStream(directory.resolve(name), StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW); + Cipher cipher = CipherFactory.getCipher(provider); + SecureRandom random = Randomness.createSecure(); + byte[] iv = new byte[CipherFactory.IV_ARRAY_LENGTH]; + random.nextBytes(iv); + CipherFactory.initCipher(cipher, this, Optional.of(iv), Cipher.ENCRYPT_MODE, 0); + ivMap.put(getDirectory() + "/" + name, Base64.getEncoder().encodeToString(iv)); + return new CryptoIndexOutput(name, fos, cipher); + } + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized void close() throws IOException { + try { + deleteFile("ivMap"); + } catch (java.nio.file.NoSuchFileException fnfe) { + + } + IndexOutput out = super.createOutput("ivMap", new IOContext()); + out.writeMapOfStrings(ivMap); + out.close(); + isOpen = false; + deletePendingFiles(); + dataKey = null; + } + + /** + * {@inheritDoc} + * @param name the name of the file to be deleted + */ + @Override + public void deleteFile(String name) throws IOException { + ivMap.remove(getDirectory() + "/" + name); + super.deleteFile(name); + } + + /** + * An IndexInput implementation that decrypts data for reading + * + * @opensearch.internal + */ + final class CryptoBufferedIndexInput extends BufferedIndexInput { + /** The maximum chunk size for reads of 16384 bytes. */ + private static final int CHUNK_SIZE = 16384; + ByteBuffer tmpBuffer = ByteBuffer.allocate(CHUNK_SIZE); + + /** the file channel we will read from */ + protected final FileChannel channel; + /** is this instance a clone and hence does not own the file to close it */ + boolean isClone = false; + /** start offset: non-zero in the slice case */ + protected final long off; + /** end offset (start+length) */ + protected final long end; + InputStream stream; + Cipher cipher; + final CryptoDirectory directory; + + public CryptoBufferedIndexInput(String resourceDesc, FileChannel fc, IOContext context, Cipher cipher, CryptoDirectory directory) + throws IOException { + super(resourceDesc, context); + this.channel = fc; + this.off = 0L; + this.end = fc.size(); + this.stream = Channels.newInputStream(channel); + this.cipher = cipher; + this.directory = directory; + } + + public CryptoBufferedIndexInput( + String resourceDesc, + FileChannel fc, + long off, + long length, + int bufferSize, + Cipher old, + CryptoDirectory directory + ) throws IOException { + super(resourceDesc, bufferSize); + this.channel = fc; + this.off = off; + this.end = off + length; + this.isClone = true; + this.directory = directory; + this.stream = Channels.newInputStream(channel); + cipher = CipherFactory.getCipher(old.getProvider()); + CipherFactory.initCipher(cipher, directory, Optional.of(old.getIV()), Cipher.DECRYPT_MODE, off); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + if (!isClone) { + stream.close(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public CryptoBufferedIndexInput clone() { + CryptoBufferedIndexInput clone = (CryptoBufferedIndexInput) super.clone(); + clone.isClone = true; + clone.cipher = CipherFactory.getCipher(cipher.getProvider()); + CipherFactory.initCipher(clone.cipher, directory, Optional.of(cipher.getIV()), Cipher.DECRYPT_MODE, getFilePointer() + off); + clone.tmpBuffer = ByteBuffer.allocate(CHUNK_SIZE); + return clone; + } + + /** + * {@inheritDoc} + */ + @Override + public IndexInput slice(String sliceDescription, long offset, long length) throws IOException { + if (offset < 0 || length < 0 || offset + length > this.length()) { + throw new IllegalArgumentException( + "slice() " + + sliceDescription + + " out of bounds: offset=" + + offset + + ",length=" + + length + + ",fileLength=" + + this.length() + + ": " + + this + ); + } + return new CryptoBufferedIndexInput( + getFullSliceDescription(sliceDescription), + channel, + off + offset, + length, + getBufferSize(), + cipher, + directory + ); + } + + /** + * {@inheritDoc} + */ + @Override + public final long length() { + return end - off; + } + + private int read(ByteBuffer dst, long position) throws IOException { + int ret; + int i; + tmpBuffer.rewind(); + // FileChannel#read is forbidden + synchronized (channel) { + channel.position(position); + i = stream.read(tmpBuffer.array(), 0, dst.remaining()); + } + tmpBuffer.limit(i); + try { + if (end - position > i) ret = cipher.update(tmpBuffer, dst); + else ret = cipher.doFinal(tmpBuffer, dst); + } catch (ShortBufferException | IllegalBlockSizeException | BadPaddingException ex) { + throw new IOException("failed to decrypt blck.", ex); + } + return ret; + } + + /** + * {@inheritDoc} + */ + @Override + protected void readInternal(ByteBuffer b) throws IOException { + long pos = getFilePointer() + off; + + if (pos + b.remaining() > end) { + throw new EOFException( + Thread.currentThread().getId() + + " read past EOF: " + + this + + " isClone? " + + isClone + + " off: " + + off + + " pos: " + + pos + + " end: " + + end + ); + } + + try { + int readLength = b.remaining(); + while (readLength > 0) { + final int toRead = Math.min(CHUNK_SIZE, readLength); + b.limit(b.position() + toRead); + assert b.remaining() == toRead; + final int i = read(b, pos); + if (i < 0) { + throw new EOFException("read past EOF: " + this + " buffer: " + b + " chunkLen: " + toRead + " end: " + end); + } + assert i > 0 : "FileChannel.read with non zero-length bb.remaining() must always read at least " + + "one byte (FileChannel is in blocking mode, see spec of ReadableByteChannel)"; + pos += i; + readLength -= i; + } + assert readLength == 0; + } catch (IOException ioe) { + throw new IOException(ioe.getMessage() + ": " + this, ioe); + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void seekInternal(long pos) throws IOException { + if (pos > length()) { + throw new EOFException( + Thread.currentThread().getId() + " read past EOF: pos=" + pos + " vs length=" + length() + ": " + this + ); + } + CipherFactory.initCipher(cipher, directory, Optional.empty(), Cipher.DECRYPT_MODE, pos + off); + } + } + + /** + * An IndexOutput implementation that encrypts data before writing + * + * @opensearch.internal + */ + final class CryptoIndexOutput extends OutputStreamIndexOutput { + /** + * The maximum chunk size is 8192 bytes, because file channel mallocs a native buffer outside of + * stack if the write buffer size is larger. + */ + static final int CHUNK_SIZE = 8192; + + public CryptoIndexOutput(String name, OutputStream os, Cipher cipher) throws IOException { + super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", name, new FilterOutputStream(os) { + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + try { + out.write(cipher.doFinal()); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new RuntimeException(e); + } + super.close(); + } + + /** + * {@inheritDoc} + */ + @Override + public void write(byte[] b, int offset, int length) throws IOException { + int count = 0; + byte[] res; + while (length > 0) { + count++; + final int chunk = Math.min(length, CHUNK_SIZE); + try { + res = cipher.update(b, offset, chunk); + if (res != null) out.write(res); + } catch (IllegalStateException e) { + throw new IllegalStateException("count is " + count + " " + e.getMessage()); + } + length -= chunk; + offset += chunk; + } + } + }, CHUNK_SIZE); + } + } + + static class CipherFactory { + static final int AES_BLOCK_SIZE_BYTES = 16; + static final int COUNTER_SIZE_BYTES = 4; + static final int IV_ARRAY_LENGTH = 16; + + public static Cipher getCipher(Provider provider) { + try { + return Cipher.getInstance("AES/CTR/NoPadding", provider); + } catch (NoSuchPaddingException | NoSuchAlgorithmException e) { + throw new RuntimeException(); + } + } + + public static void initCipher(Cipher cipher, CryptoDirectory directory, Optional ivarray, int opmode, long newPosition) { + try { + byte[] iv = ivarray.isPresent() ? ivarray.get() : cipher.getIV(); + if (newPosition == 0) { + Arrays.fill(iv, IV_ARRAY_LENGTH - COUNTER_SIZE_BYTES, IV_ARRAY_LENGTH, (byte) 0); + } else { + int counter = (int) (newPosition / AES_BLOCK_SIZE_BYTES); + for (int i = IV_ARRAY_LENGTH - 1; i >= IV_ARRAY_LENGTH - COUNTER_SIZE_BYTES; i--) { + iv[i] = (byte) counter; + counter = counter >>> Byte.SIZE; + } + } + IvParameterSpec spec = new IvParameterSpec(iv); + cipher.init(opmode, directory.dataKey, spec); + int bytesToRead = (int) (newPosition % AES_BLOCK_SIZE_BYTES); + if (bytesToRead > 0) { + cipher.update(new byte[bytesToRead]); + } + } catch (InvalidAlgorithmParameterException | InvalidKeyException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java new file mode 100644 index 0000000000000..14d6c7d0e2131 --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.store; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockFactory; +import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.common.crypto.MasterKeyProvider; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Setting.Property; +import org.opensearch.common.settings.Settings; +import org.opensearch.crypto.CryptoHandlerRegistry; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.ShardPath; +import org.opensearch.plugins.IndexStorePlugin; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.Provider; +import java.security.Security; +import java.util.function.Function; + +/** + * Factory for an encrypted filesystem directory + */ +public class CryptoDirectoryFactory implements IndexStorePlugin.DirectoryFactory { + + /** + * Creates a new CryptoDirectoryFactory + */ + public CryptoDirectoryFactory() { + super(); + } + + /** + * Specifies a crypto provider to be used for encryption. The default value is SunJCE. + */ + public static final Setting INDEX_CRYPTO_PROVIDER_SETTING = new Setting<>("index.store.crypto.provider", "SunJCE", (s) -> { + Provider p = Security.getProvider(s); + if (p == null) { + throw new IllegalArgumentException("unrecognized [index.store.crypto.provider] \"" + s + "\""); + } else return p; + }, Property.IndexScope, Property.InternalIndex); + + /** + * Specifies the Key management plugin type to be used. The desired KMS plugin should be installed. + */ + public static final Setting INDEX_KMS_TYPE_SETTING = new Setting<>( + "index.store.kms.type", + "", + Function.identity(), + Property.NodeScope, + Property.IndexScope + ); + + /** + * {@inheritDoc} + * @param indexSettings the index settings + * @param path the shard file path + */ + @Override + public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throws IOException { + final Path location = path.resolveIndex(); + final LockFactory lockFactory = indexSettings.getValue(org.opensearch.index.store.FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING); + Files.createDirectories(location); + final Provider provider = indexSettings.getValue(INDEX_CRYPTO_PROVIDER_SETTING); + final String KEY_PROVIDER_TYPE = indexSettings.getValue(INDEX_KMS_TYPE_SETTING); + final Settings settings = Settings.builder().put(indexSettings.getNodeSettings(), false).build(); + CryptoMetadata cryptoMetadata = new CryptoMetadata(KEY_PROVIDER_TYPE, KEY_PROVIDER_TYPE, settings); + MasterKeyProvider keyProvider = CryptoHandlerRegistry.getInstance() + .getCryptoKeyProviderPlugin(KEY_PROVIDER_TYPE) + .createKeyProvider(cryptoMetadata); + return new CryptoDirectory(lockFactory, location, provider, keyProvider); + } +} diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java new file mode 100644 index 0000000000000..8180ff32052ac --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.store; + +import org.opensearch.common.settings.Setting; +import org.opensearch.plugins.IndexStorePlugin; +import org.opensearch.plugins.Plugin; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * A plugin that enables index level encryption and decryption. + */ +public class CryptoDirectoryPlugin extends Plugin implements IndexStorePlugin { + + /** + * The default constructor. + */ + public CryptoDirectoryPlugin() { + super(); + } + + /** + * {@inheritDoc} + */ + @Override + public List> getSettings() { + return Arrays.asList(CryptoDirectoryFactory.INDEX_KMS_TYPE_SETTING, CryptoDirectoryFactory.INDEX_CRYPTO_PROVIDER_SETTING); + } + + /** + * {@inheritDoc} + */ + @Override + public Map getDirectoryFactories() { + return java.util.Collections.singletonMap("cryptofs", new CryptoDirectoryFactory()); + } +} diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/package-info.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/package-info.java new file mode 100644 index 0000000000000..56ba4f8f73c4f --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Encryption plugin for encrypting and decrypting index files at rest. + */ +package org.opensearch.index.store; diff --git a/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java new file mode 100644 index 0000000000000..f61464e004d58 --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java @@ -0,0 +1,165 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSLockFactory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.tests.mockfile.ExtrasFS; +import org.opensearch.common.Randomness; +import org.opensearch.common.crypto.DataKeyPair; +import org.opensearch.common.crypto.MasterKeyProvider; + +import java.io.IOException; +import java.nio.file.Path; +import java.security.Security; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * SMB Tests using NIO FileSystem as index store type. + */ +// @RunWith(RandomizedRunner.class) +public class CryptoDirectoryTests extends OpenSearchBaseDirectoryTestCase { + + static final String KEY_FILE_NAME = "keyfile"; + + /* static java.util.Random rnd; + + @BeforeClass + private static void setup() { + rnd = new java.util.Random(); //Randomness.get(); + } + */ + @Override + protected Directory getDirectory(Path file) throws IOException { + MasterKeyProvider keyProvider = mock(MasterKeyProvider.class); + byte[] rawKey = new byte[32]; + byte[] encryptedKey = new byte[32]; + java.util.Random rnd = Randomness.get(); + rnd.nextBytes(rawKey); + rnd.nextBytes(encryptedKey); + DataKeyPair dataKeyPair = new DataKeyPair(rawKey, encryptedKey); + when(keyProvider.generateDataPair()).thenReturn(dataKeyPair); + return new CryptoDirectory(FSLockFactory.getDefault(), file, Security.getProvider("SunJCE"), keyProvider); + } + + /*public void testCreateOutputForExistingFile() throws IOException { + + This test is disabled because {@link SmbDirectoryWrapper} opens existing file + with an explicit StandardOpenOption.TRUNCATE_EXISTING option. + + }*/ + + @Override + public void testCreateTempOutput() throws Throwable { + try (Directory dir = getDirectory(createTempDir())) { + List names = new ArrayList<>(); + int iters = atLeast(50); + for (int iter = 0; iter < iters; iter++) { + IndexOutput out = dir.createTempOutput("foo", "bar", newIOContext(random())); + names.add(out.getName()); + out.writeVInt(iter); + out.close(); + } + for (int iter = 0; iter < iters; iter++) { + IndexInput in = dir.openInput(names.get(iter), newIOContext(random())); + assertEquals(iter, in.readVInt()); + in.close(); + } + + Set files = Arrays.stream(dir.listAll()) + .filter(file -> !ExtrasFS.isExtra(file)) // remove any ExtrasFS stuff. + .filter(file -> !file.equals(KEY_FILE_NAME)) // remove keyfile. + .collect(Collectors.toSet()); + + assertEquals(new HashSet(names), files); + } + } + + @Override + public void testThreadSafetyInListAll() throws Exception { + /* + try (Directory dir = getDirectory(createTempDir("testThreadSafety"))) { + if (dir instanceof BaseDirectoryWrapper) { + // we are not making a real index, just writing, reading files. + ((BaseDirectoryWrapper) dir).setCheckIndexOnClose(false); + } + if (dir instanceof MockDirectoryWrapper) { + // makes this test really slow + ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); + } + + AtomicBoolean stop = new AtomicBoolean(); + Thread writer = new Thread(() -> { + try { + for (int i = 0, max = RandomizedTest.randomIntBetween(500, 1000); i < max; i++) { + String fileName = "file-" + i; + try (IndexOutput output = dir.createOutput(fileName, newIOContext(random()))) { + assert output != null; + // Add some lags so that the other thread can read the content of the + // directory. + Thread.yield(); + } + assertTrue(slowFileExists(dir, fileName)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + stop.set(true); + } + }); + + Thread reader = new Thread(() -> { + try { + Random rnd = new Random(RandomizedTest.randomLong()); + while (!stop.get()) { + String[] files = Arrays.stream(dir.listAll()) + .filter(name -> !ExtrasFS.isExtra(name)) // Ignore anything from ExtraFS. + .filter(name -> !name.equals(KEY_FILE_NAME)) // remove keyfile. + .toArray(String[]::new); + + if (files.length > 0) { + do { + String file = RandomPicks.randomFrom(rnd, files); + try (IndexInput input = dir.openInput(file, newIOContext(random()))) { + // Just open, nothing else. + assert input != null; + } catch (@SuppressWarnings("unused") AccessDeniedException e) { + // Access denied is allowed for files for which the output is still open + // (MockDirectoryWriter enforces + // this, for example). Since we don't synchronize with the writer thread, + // just ignore it. + } catch (IOException e) { + throw new UncheckedIOException("Something went wrong when opening: " + file, e); + } + } while (rnd.nextInt(3) != 0); // Sometimes break and list files again. + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + + reader.start(); + writer.start(); + + writer.join(); + reader.join(); + }*/ + } +} diff --git a/sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java b/sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..e763df565e156 --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.path.to.plugin; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + +public class CryptoDirectoryClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + + public RenameClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/sandbox/plugins/cryptodirectory/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/sandbox/plugins/cryptodirectory/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml new file mode 100644 index 0000000000000..e411271a757bf --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -0,0 +1,8 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^cryptodirectory\n$/ From 69a443b23a7d2a1b402f1d7b89eca3866c7683a2 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Mon, 25 Mar 2024 09:26:59 -0400 Subject: [PATCH 02/11] Modified changelog for PR Signed-off-by: Olasoji Denloye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 346913f025f4d..0a252b76d80a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880)) - Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103)) - Add explicit dependency to validatePom and generatePom tasks ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12807)) +- Index level encryption features (#12451) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 From eb6e4993ba7d9d5ad9a239e869538b0f648b7aa8 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Mon, 25 Mar 2024 09:33:40 -0400 Subject: [PATCH 03/11] Updated changelog with PR link Signed-off-by: Olasoji Denloye --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a252b76d80a8..21c23c0d4758b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880)) - Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103)) - Add explicit dependency to validatePom and generatePom tasks ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12807)) -- Index level encryption features (#12451) +- Index level encryption features ([#12902](https://github.com/opensearch-project/OpenSearch/pull/12902)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 From ee0b4ab77505b32e531b9fb4cd384469d945f517 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Mon, 25 Mar 2024 10:11:38 -0400 Subject: [PATCH 04/11] Fixed conflict in CHANGELOG Signed-off-by: Olasoji Denloye --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21c23c0d4758b..7501c8bc5e3c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remote reindex: Add support for configurable retry mechanism ([#12561](https://github.com/opensearch-project/OpenSearch/pull/12561)) - [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880)) - Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103)) -- Add explicit dependency to validatePom and generatePom tasks ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12807)) +- Add explicit dependency to validatePom and generatePom tasks ([#12807](https://github.com/opensearch-project/OpenSearch/pull/12807)) +- Replace configureEach with all for publication iteration ([#12876](https://github.com/opensearch-project/OpenSearch/pull/12876)) - Index level encryption features ([#12902](https://github.com/opensearch-project/OpenSearch/pull/12902)) ### Dependencies From cdfc71932009ca231042e25b475c55cea77441e5 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Tue, 30 Jul 2024 14:40:13 -0700 Subject: [PATCH 05/11] Update sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java Co-authored-by: William Beckler Signed-off-by: Olasoji Denloye --- .../index/store/CryptoDirectoryPlugin.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java index 8180ff32052ac..16bedbda5c8cb 100644 --- a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java @@ -7,23 +7,7 @@ */ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ + /* * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. From 0d802667897cefb2e4cd84313fd16372fa697a42 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Tue, 30 Jul 2024 14:40:28 -0700 Subject: [PATCH 06/11] Update sandbox/plugins/cryptodirectory/build.gradle Co-authored-by: William Beckler Signed-off-by: Olasoji Denloye --- sandbox/plugins/cryptodirectory/build.gradle | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sandbox/plugins/cryptodirectory/build.gradle b/sandbox/plugins/cryptodirectory/build.gradle index 737cdca3916c8..6e73dc15c4a0c 100644 --- a/sandbox/plugins/cryptodirectory/build.gradle +++ b/sandbox/plugins/cryptodirectory/build.gradle @@ -9,24 +9,6 @@ * GitHub history for details. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ apply plugin: 'opensearch.build' apply plugin: 'opensearch.publish' From 56d91562991f5f5eef85c275e8e76f9498baab1c Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Tue, 30 Jul 2024 14:40:40 -0700 Subject: [PATCH 07/11] Update sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java Co-authored-by: William Beckler Signed-off-by: Olasoji Denloye --- .../index/store/CryptoDirectoryFactory.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java index 14d6c7d0e2131..5e831df6fdf65 100644 --- a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java @@ -6,25 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - /* * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. From bcb359ebce17e526cd9486fa3ff43c47427e26cc Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Wed, 7 Aug 2024 19:25:23 -0700 Subject: [PATCH 08/11] - Added a few integration tests Signed-off-by: Olasoji Denloye --- sandbox/plugins/cryptodirectory/build.gradle | 8 +++ .../store/CryptoDirectoryIntegTestCase.java | 70 +++++++++++++++++++ .../index/store/CryptoDirectoryPlugin.java | 2 +- .../store/MockCryptoKeyProviderPlugin.java | 49 +++++++++++++ .../index/store/MockCryptoPlugin.java | 37 ++++++++++ .../CryptoDirectoryClientYamlTestSuiteIT.java | 4 +- 6 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java create mode 100644 sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoKeyProviderPlugin.java create mode 100644 sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoPlugin.java diff --git a/sandbox/plugins/cryptodirectory/build.gradle b/sandbox/plugins/cryptodirectory/build.gradle index 6e73dc15c4a0c..d7a0f35fbfcb0 100644 --- a/sandbox/plugins/cryptodirectory/build.gradle +++ b/sandbox/plugins/cryptodirectory/build.gradle @@ -11,11 +11,19 @@ apply plugin: 'opensearch.build' apply plugin: 'opensearch.publish' +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { description 'Encrypts and decrypts index data at rest.' classname 'org.opensearch.index.store.CryptoDirectoryPlugin' } + +dependencies { + testImplementation project(path: ':modules:reindex') + testImplementation "commons-io:commons-io:2.13.0" +} + //restResources { // restApi { // includeCore '_common', 'cluster', 'nodes', 'index', 'indices', 'get' diff --git a/sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java b/sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java new file mode 100644 index 0000000000000..b7e2a0f5e4a5e --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.store; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.reindex.ReindexAction; +import org.opensearch.index.reindex.ReindexModulePlugin; +import org.opensearch.index.reindex.ReindexRequestBuilder; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.hamcrest.Matchers.is; + +public class CryptoDirectoryIntegTestCase extends OpenSearchIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + CryptoDirectoryPlugin.class, + MockCryptoKeyProviderPlugin.class, + MockCryptoPlugin.class, + ReindexModulePlugin.class + ); + } + + @Override + public Settings indexSettings() { + return Settings.builder() + .put(super.indexSettings()) + .put("index.store.type", "cryptofs") + .put("index.store.kms.type", "dummy") + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + } + + public void testReindex() { + // Create an index and index some documents + createIndex("test"); + createIndex("test_copy"); + long nbDocs = randomIntBetween(10, 1000); + for (long i = 0; i < nbDocs; i++) { + index("test", "doc", "" + i, "foo", "bar"); + } + refresh(); + SearchResponse response = client().prepareSearch("test").get(); + assertThat(response.getHits().getTotalHits().value, is(nbDocs)); + + // Reindex + reindex().source("test").destination("test_copy").refresh(true).get(); + SearchResponse copy_response = client().prepareSearch("test_copy").get(); + assertThat(copy_response.getHits().getTotalHits().value, is(nbDocs)); + + } + + ReindexRequestBuilder reindex() { + return new ReindexRequestBuilder(client(), ReindexAction.INSTANCE); + } + +} diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java index 16bedbda5c8cb..0c6d89c73d5f8 100644 --- a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java @@ -7,7 +7,7 @@ */ /* - + /* * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. diff --git a/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoKeyProviderPlugin.java b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoKeyProviderPlugin.java new file mode 100644 index 0000000000000..ff4389410875a --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoKeyProviderPlugin.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.store; + +import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.common.Randomness; +import org.opensearch.common.crypto.DataKeyPair; +import org.opensearch.common.crypto.MasterKeyProvider; +import org.opensearch.plugins.CryptoKeyProviderPlugin; +import org.opensearch.plugins.Plugin; + +import static org.mockito.AdditionalAnswers.returnsFirstArg; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Some tests rely on the keyword tokenizer, but this tokenizer isn't part of lucene-core and therefor not available + * in some modules. What this test plugin does, is use the mock tokenizer and advertise that as the keyword tokenizer. + *

+ * Most tests that need this test plugin use normalizers. When normalizers are constructed they try to resolve the + * keyword tokenizer, but if the keyword tokenizer isn't available then constructing normalizers will fail. + */ +public class MockCryptoKeyProviderPlugin extends Plugin implements CryptoKeyProviderPlugin { + + @Override + public MasterKeyProvider createKeyProvider(CryptoMetadata cryptoMetadata) { + MasterKeyProvider keyProvider = mock(MasterKeyProvider.class); + byte[] rawKey = new byte[32]; + byte[] encryptedKey = new byte[32]; + java.util.Random rnd = Randomness.get(); + rnd.nextBytes(rawKey); + rnd.nextBytes(encryptedKey); + DataKeyPair dataKeyPair = new DataKeyPair(rawKey, encryptedKey); + when(keyProvider.generateDataPair()).thenReturn(dataKeyPair); + when(keyProvider.decryptKey(any(byte[].class))).then(returnsFirstArg()); + return keyProvider; + } + + @Override + public String type() { + return "dummy"; + } +} diff --git a/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoPlugin.java b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoPlugin.java new file mode 100644 index 0000000000000..785189b12ca56 --- /dev/null +++ b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/MockCryptoPlugin.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.store; + +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.MasterKeyProvider; +import org.opensearch.plugins.CryptoPlugin; +import org.opensearch.plugins.Plugin; + +import static org.mockito.Mockito.mock; + +/** + * Some tests rely on the keyword tokenizer, but this tokenizer isn't part of lucene-core and therefor not available + * in some modules. What this test plugin does, is use the mock tokenizer and advertise that as the keyword tokenizer. + *

+ * Most tests that need this test plugin use normalizers. When normalizers are constructed they try to resolve the + * keyword tokenizer, but if the keyword tokenizer isn't available then constructing normalizers will fail. + */ +public class MockCryptoPlugin extends Plugin implements CryptoPlugin { + + @Override + @SuppressWarnings("unchecked") + public CryptoHandler getOrCreateCryptoHandler( + MasterKeyProvider keyProvider, + String keyProviderName, + String keyProviderType, + Runnable onClose + ) { + CryptoHandler handler = (CryptoHandler) mock(CryptoHandler.class); + return handler; + } +} diff --git a/sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java b/sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java index e763df565e156..7bd5a409f7a8f 100644 --- a/sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java +++ b/sandbox/plugins/cryptodirectory/src/yamlRestTest/java/org/opensearch/index/store/CryptoDirectoryClientYamlTestSuiteIT.java @@ -5,7 +5,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.path.to.plugin; +package org.opensearch.index.store; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -15,7 +15,7 @@ public class CryptoDirectoryClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { - public RenameClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + public CryptoDirectoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } From 198deb2030e7db12de76cbf8e4cba69f0980822e Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Thu, 29 Aug 2024 16:49:33 -0700 Subject: [PATCH 09/11] - Added more integration tests and fixed issues to resolve PR review comments Signed-off-by: Olasoji Denloye --- sandbox/plugins/cryptodirectory/build.gradle | 6 -- .../store/CryptoDirectoryIntegTestCase.java | 59 ++++++++++++++++++- .../index/store/CryptoDirectoryFactory.java | 38 +++++++----- .../index/store/CryptoDirectoryTests.java | 16 +---- 4 files changed, 82 insertions(+), 37 deletions(-) diff --git a/sandbox/plugins/cryptodirectory/build.gradle b/sandbox/plugins/cryptodirectory/build.gradle index d7a0f35fbfcb0..4616effcc57bd 100644 --- a/sandbox/plugins/cryptodirectory/build.gradle +++ b/sandbox/plugins/cryptodirectory/build.gradle @@ -23,9 +23,3 @@ dependencies { testImplementation project(path: ':modules:reindex') testImplementation "commons-io:commons-io:2.13.0" } - -//restResources { -// restApi { -// includeCore '_common', 'cluster', 'nodes', 'index', 'indices', 'get' -// } -//} diff --git a/sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java b/sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java index b7e2a0f5e4a5e..b957b3bb1bce7 100644 --- a/sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java +++ b/sandbox/plugins/cryptodirectory/src/internalClusterTest/java/org/opensearch/index/store/CryptoDirectoryIntegTestCase.java @@ -7,8 +7,10 @@ */ package org.opensearch.index.store; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.reindex.ReindexAction; import org.opensearch.index.reindex.ReindexModulePlugin; import org.opensearch.index.reindex.ReindexRequestBuilder; @@ -39,9 +41,46 @@ public Settings indexSettings() { .put(super.indexSettings()) .put("index.store.type", "cryptofs") .put("index.store.kms.type", "dummy") + .build(); + } + + public void testEmptyStoreTypeSettings() { + Settings settings = Settings.builder() + .put(super.indexSettings()) + .put("index.store.type", "cryptofs") .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 0) .build(); + + // Create an index and index some documents + createIndex("test", settings); + long nbDocs = randomIntBetween(10, 1000); + final Exception e = expectThrows(Exception.class, () -> { + for (long i = 0; i < nbDocs; i++) { + index("test", "doc", "" + i, "foo", "bar"); + } + }); + assertTrue(e instanceof Exception); + } + + public void testUnavailableStoreType() { + Settings settings = Settings.builder() + .put(super.indexSettings()) + .put("index.store.type", "cryptofs") + .put("index.store.kms.type", "unavailable") + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + + // Create an index and index some documents + createIndex("test", settings); + long nbDocs = randomIntBetween(10, 1000); + final Exception e = expectThrows(Exception.class, () -> { + for (long i = 0; i < nbDocs; i++) { + index("test", "doc", "" + i, "foo", "bar"); + } + }); + assertTrue(e instanceof Exception); } public void testReindex() { @@ -63,8 +102,26 @@ public void testReindex() { } + public void testDelete() { + // Create an index and index some documents + createIndex("todelete"); + long nbDocs = randomIntBetween(10, 1000); + for (long i = 0; i < nbDocs; i++) { + index("todelete", "doc", "" + i, "foo", "bar"); + } + refresh(); + SearchResponse response = client().prepareSearch("todelete").get(); + assertThat(response.getHits().getTotalHits().value, is(nbDocs)); + + // Deleteindex + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest("todelete"); + client().admin().indices().delete(deleteIndexRequest).actionGet(); + + final IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> client().prepareSearch("todelete").get()); + assertTrue(e.getMessage().contains("no such index")); + } + ReindexRequestBuilder reindex() { return new ReindexRequestBuilder(client(), ReindexAction.INSTANCE); } - } diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java index 5e831df6fdf65..9b648dba5b32d 100644 --- a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectoryFactory.java @@ -20,6 +20,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsException; import org.opensearch.crypto.CryptoHandlerRegistry; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; @@ -50,20 +51,33 @@ public CryptoDirectoryFactory() { public static final Setting INDEX_CRYPTO_PROVIDER_SETTING = new Setting<>("index.store.crypto.provider", "SunJCE", (s) -> { Provider p = Security.getProvider(s); if (p == null) { - throw new IllegalArgumentException("unrecognized [index.store.crypto.provider] \"" + s + "\""); + throw new SettingsException("unrecognized [index.store.crypto.provider] \"" + s + "\""); } else return p; }, Property.IndexScope, Property.InternalIndex); /** * Specifies the Key management plugin type to be used. The desired KMS plugin should be installed. */ - public static final Setting INDEX_KMS_TYPE_SETTING = new Setting<>( - "index.store.kms.type", - "", - Function.identity(), - Property.NodeScope, - Property.IndexScope - ); + public static final Setting INDEX_KMS_TYPE_SETTING = new Setting<>("index.store.kms.type", "", Function.identity(), (s) -> { + if (s == null || s.isEmpty()) { + throw new SettingsException("index.store.kms.type must be set"); + } + }, Property.NodeScope, Property.IndexScope); + + MasterKeyProvider getKeyProvider(IndexSettings indexSettings) { + final String KEY_PROVIDER_TYPE = indexSettings.getValue(INDEX_KMS_TYPE_SETTING); + final Settings settings = Settings.builder().put(indexSettings.getNodeSettings(), false).build(); + CryptoMetadata cryptoMetadata = new CryptoMetadata("", KEY_PROVIDER_TYPE, settings); + MasterKeyProvider keyProvider; + try { + keyProvider = CryptoHandlerRegistry.getInstance() + .getCryptoKeyProviderPlugin(KEY_PROVIDER_TYPE) + .createKeyProvider(cryptoMetadata); + } catch (NullPointerException npe) { + throw new RuntimeException("could not find key provider: " + KEY_PROVIDER_TYPE, npe); + } + return keyProvider; + } /** * {@inheritDoc} @@ -76,12 +90,6 @@ public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throw final LockFactory lockFactory = indexSettings.getValue(org.opensearch.index.store.FsDirectoryFactory.INDEX_LOCK_FACTOR_SETTING); Files.createDirectories(location); final Provider provider = indexSettings.getValue(INDEX_CRYPTO_PROVIDER_SETTING); - final String KEY_PROVIDER_TYPE = indexSettings.getValue(INDEX_KMS_TYPE_SETTING); - final Settings settings = Settings.builder().put(indexSettings.getNodeSettings(), false).build(); - CryptoMetadata cryptoMetadata = new CryptoMetadata(KEY_PROVIDER_TYPE, KEY_PROVIDER_TYPE, settings); - MasterKeyProvider keyProvider = CryptoHandlerRegistry.getInstance() - .getCryptoKeyProviderPlugin(KEY_PROVIDER_TYPE) - .createKeyProvider(cryptoMetadata); - return new CryptoDirectory(lockFactory, location, provider, keyProvider); + return new CryptoDirectory(lockFactory, location, provider, getKeyProvider(indexSettings)); } } diff --git a/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java index f61464e004d58..f69b21536b8ba 100644 --- a/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java +++ b/sandbox/plugins/cryptodirectory/src/test/java/org/opensearch/index/store/CryptoDirectoryTests.java @@ -38,13 +38,6 @@ public class CryptoDirectoryTests extends OpenSearchBaseDirectoryTestCase { static final String KEY_FILE_NAME = "keyfile"; - /* static java.util.Random rnd; - - @BeforeClass - private static void setup() { - rnd = new java.util.Random(); //Randomness.get(); - } - */ @Override protected Directory getDirectory(Path file) throws IOException { MasterKeyProvider keyProvider = mock(MasterKeyProvider.class); @@ -58,13 +51,6 @@ protected Directory getDirectory(Path file) throws IOException { return new CryptoDirectory(FSLockFactory.getDefault(), file, Security.getProvider("SunJCE"), keyProvider); } - /*public void testCreateOutputForExistingFile() throws IOException { - - This test is disabled because {@link SmbDirectoryWrapper} opens existing file - with an explicit StandardOpenOption.TRUNCATE_EXISTING option. - - }*/ - @Override public void testCreateTempOutput() throws Throwable { try (Directory dir = getDirectory(createTempDir())) { @@ -160,6 +146,6 @@ public void testThreadSafetyInListAll() throws Exception { writer.join(); reader.join(); - }*/ + } */ } } From 6dbf41d8a3dad08a3c501542b4f7f28ac84d4d54 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Fri, 8 Nov 2024 20:23:16 -0800 Subject: [PATCH 10/11] Replaces synchronized block with FileChannel#read Signed-off-by: Olasoji Denloye --- .../org/opensearch/index/store/CryptoDirectory.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java index 2189f29bf6811..2a9039883756a 100644 --- a/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java +++ b/sandbox/plugins/cryptodirectory/src/main/java/org/opensearch/index/store/CryptoDirectory.java @@ -38,6 +38,7 @@ import org.apache.lucene.store.NIOFSDirectory; import org.apache.lucene.store.OutputStreamIndexOutput; import org.opensearch.common.Randomness; +import org.opensearch.common.SuppressForbidden; import org.opensearch.common.crypto.DataKeyPair; import org.opensearch.common.crypto.MasterKeyProvider; import org.opensearch.common.util.io.IOUtils; @@ -362,16 +363,21 @@ public final long length() { return end - off; } + @SuppressForbidden(reason = "FileChannel#read is a faster alternative to synchronized block") private int read(ByteBuffer dst, long position) throws IOException { int ret; int i; - tmpBuffer.rewind(); + tmpBuffer.rewind().limit(dst.remaining()); + /* tmpBuffer.rewind(); // FileChannel#read is forbidden - synchronized (channel) { + /* synchronized (channel) { channel.position(position); i = stream.read(tmpBuffer.array(), 0, dst.remaining()); } tmpBuffer.limit(i); + */ + i = channel.read(tmpBuffer, position); + tmpBuffer.flip(); try { if (end - position > i) ret = cipher.update(tmpBuffer, dst); else ret = cipher.doFinal(tmpBuffer, dst); From ded0f58e86bf92a5d3d584ba7b30d6b69547c856 Mon Sep 17 00:00:00 2001 From: Olasoji Denloye Date: Thu, 5 Dec 2024 10:03:27 -0800 Subject: [PATCH 11/11] Reverted gradle properties file Signed-off-by: Olasoji Denloye --- gradle.properties | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gradle.properties b/gradle.properties index 1d1440c35c9fe..d4634f1a7aeea 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,7 +35,3 @@ systemProp.jdk.tls.client.protocols=TLSv1.2,TLSv1.3 # jvm args for faster test execution by default systemProp.tests.jvm.argline=-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m -systemProp.http.proxyHost=proxy.jf.intel.com -systemProp.http.proxyPort=911 -systemProp.https.proxyHost=proxy.jf.intel.com -systemProp.https.proxyPort=911