Skip to content

Commit

Permalink
Merge pull request Azure#3 from asorrin-msft/encryption
Browse files Browse the repository at this point in the history 
Encryption
  • Loading branch information
@emgerner-msft
emgerner-msft committed Mar 24, 2016
2 parents ff40cd1 + 60e2f3c commit 818759a
Show file tree
Hide file tree
Showing 8 changed files with 337 additions and 21 deletions.
4 changes: 3 additions & 1 deletion ChangeLog.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
2015.10.05 Version 4.0-alpha-1
* Added preview support for client side encryption for blobs, queues and tables.
* Added support for client side encryption for blobs, queues and tables.
* Since the encryption preview, added functionality where uploading encrypted blobs can be done with just PutBlob, not PutBlock + PutBlockList, if the blob is small enough.
* Since the encryption preview, fixed bugs in the Table Service where APIs such as 'CreateTable' were trying to encrypt their payload. Encryption is only supported on entities.

2015.10.05 Version 4.0.0
* Removed deprecated table AtomPub support.
Expand Down
115 changes: 115 additions & 0 deletions ...oft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlobEncryptionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;

import org.apache.commons.lang.mutable.MutableInt;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -47,6 +48,8 @@
import com.microsoft.azure.storage.Constants;
import com.microsoft.azure.storage.DictionaryKeyResolver;
import com.microsoft.azure.storage.OperationContext;
import com.microsoft.azure.storage.SendingRequestEvent;
import com.microsoft.azure.storage.StorageEvent;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.TestHelper;
import com.microsoft.azure.storage.TestRunners.CloudTests;
Expand Down Expand Up @@ -895,4 +898,116 @@ public void testBlobEncryptionWithStrictModeOnPartialBlob() throws URISyntaxExce
assertEquals(ex.getMessage(), SR.ENCRYPTION_NOT_SUPPORTED_FOR_OPERATION);
}
}

@Test
public void testBlockBlobEncryptionCountOperationsEncryptCalculateMD5PassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(true, true, true);
}

@Test
public void testBlockBlobEncryptionCountOperationsEncryptCalculateMD5NoPassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(true, true, false);
}

@Test
public void testBlockBlobEncryptionCountOperationsEncryptNoCalculateMD5PassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(true, false, true);
}

@Test
public void testBlockBlobEncryptionCountOperationsEncryptNoCalculateMD5NoPassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(true, false, false);
}

@Test
public void testBlockBlobEncryptionCountOperationsNoEncryptCalculateMD5PassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(false, true, true);
}

@Test
public void testBlockBlobEncryptionCountOperationsNoEncryptCalculateMD5NoPassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(false, true, false);
}

@Test
public void testBlockBlobEncryptionCountOperationsNoEncryptNoCalculateMD5PassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(false, false, true);
}

@Test
public void testBlockBlobEncryptionCountOperationsNoEncryptNoCalculateMD5NoPassInLength() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.runBlockBlobEncryptionTests(false, false, false);
}

public void runBlockBlobEncryptionTests(boolean encryptData, boolean calculateMD5, boolean passInLength) throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException, IOException {
this.doEncryptionTestCountOperations(0, 1, encryptData, calculateMD5, passInLength); // Test the zero-byte case
this.doEncryptionTestCountOperations(10, 1, encryptData, calculateMD5, passInLength); // Test a case that should definitely fit in one put blob, and is not 16-byte aligned.
this.doEncryptionTestCountOperations(1 * Constants.MB, 1, encryptData, calculateMD5, passInLength); // Test a case that is 16-byte aligned, and should fit in one put blob
this.doEncryptionTestCountOperations(13 * Constants.MB, 5, encryptData, calculateMD5, passInLength); // Test a case that should not hit put blob, but instead several put block + put block list.
}

private void doEncryptionTestCountOperations(int size, int count, boolean encryptData, boolean calculateMD5, boolean passInLength) throws URISyntaxException, StorageException, InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, IOException
{
byte[] buffer = BlobTestHelper.getRandomBuffer(size);

CloudBlockBlob blob = this.container.getBlockBlobReference("blockblob");

// Create the Key to be used for wrapping.
SymmetricKey aesKey = TestHelper.getSymmetricKey();

// Create the resolver to be used for unwrapping.
DictionaryKeyResolver resolver = null;

// Set the encryption policy on the request options.
BlobRequestOptions uploadOptions = new BlobRequestOptions();
if (encryptData) {
resolver = new DictionaryKeyResolver();
resolver.add(aesKey);

// Create the encryption policy to be used for upload.
BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(aesKey, null);
uploadOptions.setEncryptionPolicy(uploadPolicy);
}

uploadOptions.setStoreBlobContentMD5(calculateMD5);
uploadOptions.setUseTransactionalContentMD5(calculateMD5);
uploadOptions.setDisableContentMD5Validation(!calculateMD5);
uploadOptions.setSingleBlobPutThresholdInBytes(8 * Constants.MB);
blob.setStreamWriteSizeInBytes(4 * Constants.MB);

OperationContext opContext = new OperationContext();

final MutableInt operationCount = new MutableInt(0);

opContext.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() {

@Override
public void eventOccurred(SendingRequestEvent eventArg) {
operationCount.increment();
}
});

// Upload the encrypted contents to the blob.
ByteArrayInputStream stream = new ByteArrayInputStream(buffer);
blob.upload(stream, passInLength ? size : -1, null, uploadOptions, opContext);
assertEquals(operationCount.intValue(), count);

// Set the decryption policy on the request options.
BlobRequestOptions downloadOptions = new BlobRequestOptions();
if (encryptData) {
// Download the encrypted blob.
// Create the decryption policy to be used for download. There is no need to specify the
// key when the policy is only going to be used for downloads. Resolver is sufficient.
BlobEncryptionPolicy downloadPolicy = new BlobEncryptionPolicy(null, resolver);
downloadOptions.setEncryptionPolicy(downloadPolicy);
}

// Download and decrypt the encrypted contents from the blob.
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
blob.download(outputStream, null, downloadOptions, null);

// Compare that the decrypted contents match the input data.
TestHelper.assertStreamsAreEqualAtIndex(stream, new ByteArrayInputStream(outputStream.toByteArray()), 0, 0,
size, 2 * 1024);
}
}
71 changes: 71 additions & 0 deletions microsoft-azure-storage-test/src/com/microsoft/azure/storage/table/TableEncryptionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import com.microsoft.azure.keyvault.extensions.SymmetricKey;
import com.microsoft.azure.storage.Constants;
import com.microsoft.azure.storage.DictionaryKeyResolver;
import com.microsoft.azure.storage.ResultContinuation;
import com.microsoft.azure.storage.ResultSegment;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.TestHelper;
import com.microsoft.azure.storage.core.SR;
Expand Down Expand Up @@ -1019,6 +1021,73 @@ public void testTableOperationEncryptionWithStrictModeOnMerge() throws InvalidKe
}
}

@Test
public void testTableOperationsIgnoreEncryption() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException, URISyntaxException, StorageException
{
SymmetricKey aesKey = TestHelper.getSymmetricKey();
TableRequestOptions options = new TableRequestOptions();
options.setEncryptionPolicy(new TableEncryptionPolicy(aesKey, null));
options.setRequireEncryption(true);

CloudTableClient tableClient = TableTestHelper.createCloudTableClient();
CloudTable testTable = TableTestHelper.getRandomTableReference();

try
{
// Check Create()
testTable.create(options, null);
assertTrue("Table failed to be created when encryption policy was supplied.", testTable.exists());

// Check Exists()
assertTrue("Table.Exists() failed when encryption policy was supplied.", testTable.exists(options, null));

// Check ListTables()
for (String tableName : tableClient.listTables(testTable.getName(), options, null))
{
assertEquals("ListTables failed when an encryption policy was specified.", testTable.getName(), tableName);
}

// Check ListTablesSegmented()
for (String tableName : this.listAllTables(tableClient, testTable.getName(), options))
{
assertEquals("ListTables failed when an encryption policy was specified.", testTable.getName(), tableName);
}

// Check Get and Set Permissions
TablePermissions permissions = testTable.downloadPermissions();
String policyName = "samplePolicy";
SharedAccessTablePolicy tempPolicy = new SharedAccessTablePolicy();
tempPolicy.setPermissionsFromString("r");
tempPolicy.setSharedAccessExpiryTime(new Date());
permissions.getSharedAccessPolicies().put(policyName, tempPolicy);
testTable.uploadPermissions(permissions, options, null);
assertTrue(testTable.downloadPermissions().getSharedAccessPolicies().containsKey(policyName));
assertTrue(testTable.downloadPermissions(options, null).getSharedAccessPolicies().containsKey(policyName));

// Check Delete
testTable.delete(options, null);
assertFalse(testTable.exists());
}
finally
{
testTable.deleteIfExists();
}
}

private ArrayList<String> listAllTables(CloudTableClient tableClient, String prefix, TableRequestOptions options) throws StorageException
{
ResultContinuation token = null;
ArrayList<String> tables = new ArrayList<String>();

do
{
ResultSegment<String> tableSegment = tableClient.listTablesSegmented(prefix, null, token, options, null);
tables.addAll(tableSegment.getResults());
token = tableSegment.getContinuationToken();
} while (token != null);
return tables;
}

private static DynamicTableEntity generateRandomEntity(String pk) {
DynamicTableEntity ent = new DynamicTableEntity();
ent.getProperties().put("foo", new EntityProperty("bar"));
Expand All @@ -1028,3 +1097,5 @@ private static DynamicTableEntity generateRandomEntity(String pk) {
return ent;
}
}


63 changes: 49 additions & 14 deletions microsoft-azure-storage/src/com/microsoft/azure/storage/blob/CloudBlockBlob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -643,43 +643,78 @@ public void upload(final InputStream sourceStream, final long length, final Acce

StreamMd5AndLength descriptor = new StreamMd5AndLength();
descriptor.setLength(length);

InputStream inputDataStream = sourceStream;

// Initial check - skip the PutBlob operation if the input stream isn't markable, or if the length is known to
// be greater than the threshold.
boolean skipPutBlob = !inputDataStream.markSupported() || descriptor.getLength() > options.getSingleBlobPutThresholdInBytes();

if (sourceStream.markSupported()) {
if (inputDataStream.markSupported()) {
// Mark sourceStream for current position.
sourceStream.mark(Constants.MAX_MARK_LENGTH);
inputDataStream.mark(Constants.MAX_MARK_LENGTH);
}

// If the stream is rewindable and the length is unknown or we need to
// If we're not yet skipping PutBlob and we need to encrypt, encrypt the data and check that the encrypted
// data is under the threshold.
// Note this will abort at
// options.getSingleBlobPutThresholdInBytes() bytes and return -1.
if (!skipPutBlob && options.getEncryptionPolicy() != null)
{
class GettableByteArrayOutputStream extends ByteArrayOutputStream {
public byte[] getByteArray() { return this.buf; }
}

Cipher cipher = options.getEncryptionPolicy().createAndSetEncryptionContext(this.getMetadata(), false /* noPadding */);
GettableByteArrayOutputStream targetStream = new GettableByteArrayOutputStream();
long byteCount = Utility.encryptStreamIfUnderThreshold(inputDataStream, targetStream, cipher, descriptor.getLength(),
options.getSingleBlobPutThresholdInBytes() + 1 /*abandon if the operation hits this limit*/);

if (byteCount >= 0)
{
inputDataStream = new ByteArrayInputStream(targetStream.getByteArray());
descriptor.setLength(byteCount);
}
else
{
// If the encrypted data is over the threshold, skip PutBlob.
skipPutBlob = true;
}
}

// If we're not yet skipping PutBlob, and the length is still unknown or we need to
// set md5, then analyze the stream.
// Note this read will abort at
// options.getSingleBlobPutThresholdInBytes() bytes and return
// -1 as length in which case we will revert to using a stream as it is
// over the single put threshold.
if (sourceStream.markSupported()
&& (length < 0 || (options.getStoreBlobContentMD5() && length <= options
.getSingleBlobPutThresholdInBytes()))) {
if (!skipPutBlob && (descriptor.getLength() < 0 || options.getStoreBlobContentMD5())) {
// If the stream is of unknown length or we need to calculate
// the MD5, then we we need to read the stream contents first

descriptor = Utility.analyzeStream(sourceStream, length, options.getSingleBlobPutThresholdInBytes() + 1,
descriptor = Utility.analyzeStream(inputDataStream, descriptor.getLength(),
options.getSingleBlobPutThresholdInBytes() + 1 /*abandon if the operation hits this limit*/,
true /* rewindSourceStream */, options.getStoreBlobContentMD5());

if (descriptor.getMd5() != null && options.getStoreBlobContentMD5()) {
this.properties.setContentMD5(descriptor.getMd5());
}

// If the data is over the threshold, skip PutBlob.
if (descriptor.getLength() == -1 || descriptor.getLength() > options.getSingleBlobPutThresholdInBytes())
{
skipPutBlob = true;
}
}

// If the stream is rewindable, and the length is known and less than
// threshold the upload in a single put, otherwise use a stream.
if (sourceStream.markSupported() && descriptor.getLength() != -1
&& descriptor.getLength() < options.getSingleBlobPutThresholdInBytes() + 1
&& options.getEncryptionPolicy() == null) {
this.uploadFullBlob(sourceStream, descriptor.getLength(), accessCondition, options, opContext);
// By now, the skipPutBlob is completely correct.
if (!skipPutBlob) {
this.uploadFullBlob(inputDataStream, descriptor.getLength(), accessCondition, options, opContext);
}
else {
final BlobOutputStream writeStream = this.openOutputStream(accessCondition, options, opContext);
try {
writeStream.write(sourceStream, length);
writeStream.write(inputDataStream, length);
}
finally {
writeStream.close();
Expand Down
Loading

0 comments on commit 818759a

Please sign in to comment.