Skip to content

Commit

Permalink
Add S3 SSE KMS key and bucketked encryption
Browse files Browse the repository at this point in the history 
Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Keep miemtype handling for writeObject

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Fix syntax problem

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Fix wrong class reference name to make test run

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Remove redundant checking

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Prohibit fragments for empty streams

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Change fragment handling strategy for empty files

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Add empty file test

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Add lacking cleanup for some objects created.

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Clean up with fixer

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Backport key handling fix from s3ssekms

Reduce sending of kms key to minimum in requests

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Correct bucket key headers

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Experiment with ObjectUploader

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Disable object uploader experiment

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Fix crypto headers for empty files (backportproblem)

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Fix encryption params on empty files

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

Avoid AbortMultiPartUpload if UploadId is lacking

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
  • Loading branch information
@juliusknorr
tsdicloud authored and juliusknorr committed Aug 26, 2021
1 parent 98ad69f commit 7fa38f7
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 25 deletions.
152 changes: 131 additions & 21 deletions lib/private/Files/ObjectStore/S3ConnectionTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
use Aws\S3\S3Client;
use GuzzleHttp\Promise;
use GuzzleHttp\Promise\RejectedPromise;
use OCP\ILogger;
use Psr\Log\LoggerInterface;

trait S3ConnectionTrait {
/** @var array */
Expand All @@ -63,11 +63,17 @@ trait S3ConnectionTrait {
/** @var int */
protected $uploadPartSize;

/** @var string */
protected $sseKmsKeyId;

/** @var bool */
protected $sseUseBucketKey;

protected $test;

protected function parseParams($params) {
if (empty($params['bucket'])) {
throw new \Exception("Bucket has to be configured.");
throw new \Exception('Bucket has to be configured.');
}

$this->id = 'amazon::' . $params['bucket'];
Expand All @@ -82,7 +88,11 @@ protected function parseParams($params) {
if (!isset($params['port']) || $params['port'] === '') {
$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
}
$params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists'];
$params['autocreate'] = $params['autocreate'] ?? false;

$this->sseKmsKeyId = $params['ssekmskeyid'] ?? null;
$this->sseUseBucketKey = $params['sseusebucketkey'] ?? false;

$this->params = $params;
}

Expand All @@ -95,7 +105,118 @@ public function getProxy() {
}

/**
* Returns the connection
* Add the SSE KMS parameterdepending on the
* KMS encryption strategy (bucket, individual or
* no encryption) for object creations.
*
* @return array with encryption parameters
*/
public function getSseKmsPutParameters(): array {
if ($this->sseUseBucketKey) {
return [
'ServerSideEncryption' => 'aws:kms',
'BucketKeyEnabled' => true,
];
}

if (!empty($this->sseKmsKeyId)) {
return [
'ServerSideEncryption' => 'aws:kms',
'BucketKeyEnabled' => false,
'SSEKMSKeyId' => $this->sseKmsKeyId,
];
}

return [];
}

/**
* Add the SSE KMS parameter depending on the
* KMS encryption strategy (bucket, individual or
* no encryption) for object read.
*
* @return array with encryption parameters
*/
public function getSseKmsGetParameters(): array {
if ($this->sseUseBucketKey || !empty($this->sseKmsKeyId)) {
return [
'ServerSideEncryption' => 'aws:kms',
];
}
return [];
}

/**
* Create the required bucket
*
* @throws \Exception if bucket creation fails
*/
protected function createNewBucket() {
$logger = \OC::$server->get(LoggerInterface::class);
try {
$logger->info('Bucket "'.$this->bucket.'" does not exist - creating it.', ['app' => 'objectstore']);
if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
throw new \Exception('The bucket will not be created because the name is not dns compatible, please correct it: '.$this->bucket);
}
$this->connection->createBucket(['Bucket' => $this->bucket]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->error('Invalid remote storage.', [
'exception' => $e,
'app' => 'objectstore',
]);
throw new \Exception('Creation of bucket "'.$this->bucket.'" failed. '.$e->getMessage());
}
}

/**
* Check bucket key consistency or put bucket key if missing
* This operation only works for bucket owner or with
* s3:GetEncryptionConfiguration/s3:PutEncryptionConfiguration permission
*
* We recommend to use autocreate only on initial setup and
* use an S3:user only with object operation permission and no bucket operation permissions
* later with autocreate=false
*
* @throws \Exception if bucket key config is inconsistent or if putting the key fails
*/
protected function checkOrPutBucketKey() {
$logger = \OC::$server->get(LoggerInterface::class);

try {
$encryptState = $this->connection->getBucketEncryption([
'Bucket' => $this->bucket,
]);
} catch (S3Exception $e) {
try {
$logger->info('Bucket key for "'.$this->bucket.'" is not set - adding it.', ['app' => 'objectstore']);
$this->connection->putBucketEncryption([
'Bucket' => $this->bucket,
'ServerSideEncryptionConfiguration' => [
'Rules' => [
[
'ApplyServerSideEncryptionByDefault' => [
'SSEAlgorithm' => 'aws:kms',
'KMSMasterKeyID' => $this->sseKmsKeyId,
],
],
],
],
]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->error('Bucket key problem.', [
'exception' => $e,
'app' => 'objectstore',
]);
throw new \Exception('Putting configured bucket key to "' . $this->bucket . '" failed. ' . $e->getMessage());
}
}
}


/**
* Returns the connection.
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
Expand Down Expand Up @@ -145,23 +266,12 @@ public function getConnection() {
['app' => 'objectstore']);
}

if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) {
$logger = \OC::$server->getLogger();
try {
$logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket);
}
$this->connection->createBucket(['Bucket' => $this->bucket]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->logException($e, [
'message' => 'Invalid remote storage.',
'level' => ILogger::DEBUG,
'app' => 'objectstore',
]);
throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
}
if ($this->params['autocreate'] && !$this->connection->doesBucketExist($this->bucket)) {
$this->createNewBucket();
}

if ($this->params['autocreate'] && $this->sseUseBucketKey) {
$this->checkOrPutBucketKey();
}

// google cloud's s3 compatibility doesn't like the EncodingType parameter
Expand Down
16 changes: 12 additions & 4 deletions lib/private/Files/ObjectStore/S3ObjectTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;

Expand All @@ -43,6 +42,12 @@ trait S3ObjectTrait {
*/
abstract protected function getConnection();

/* compute configured encryption headers for put operations */
abstract protected function getSseKmsPutParameters();

/* compute configured encryption headers for get operations */
abstract protected function getSseKmsGetParameters();

/**
* @param string $urn the unified resource name used to identify the object
* @return resource stream with the read data
Expand All @@ -55,7 +60,7 @@ public function readObject($urn) {
'Bucket' => $this->bucket,
'Key' => $urn,
'Range' => 'bytes=' . $range,
]);
] + $this->getSseKmsGetParameters());
$request = \Aws\serialize($command);
$headers = [];
foreach ($request->getHeaders() as $key => $values) {
Expand Down Expand Up @@ -95,6 +100,7 @@ protected function writeSingle(string $urn, StreamInterface $stream, string $mim
'Body' => $stream,
'ACL' => 'private',
'ContentType' => $mimetype,
'params' => $this->getSseKmsPutParameters()
]);
}

Expand All @@ -114,7 +120,7 @@ protected function writeMultiPart(string $urn, StreamInterface $stream, string $
'part_size' => $this->uploadPartSize,
'params' => [
'ContentType' => $mimetype
],
] + $this->getSseKmsPutParameters(),
]);

try {
Expand Down Expand Up @@ -173,6 +179,8 @@ public function objectExists($urn) {
}

public function copyObject($from, $to) {
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to);
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [
'params' => $this->getSseKmsPutParameters(),
]);
}
}

0 comments on commit 7fa38f7

Please sign in to comment.