From 67e17ded76ccf182652e15af7184ca1cb237e540 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sat, 16 Mar 2024 21:18:44 +0100 Subject: [PATCH] feat(ObjectStore): Make S3 MultipartUpload concurrency configurable Signed-off-by: Ferdinand Thiessen --- lib/private/Files/ObjectStore/S3ConfigTrait.php | 3 +++ lib/private/Files/ObjectStore/S3ConnectionTrait.php | 8 +++++--- lib/private/Files/ObjectStore/S3ObjectTrait.php | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/private/Files/ObjectStore/S3ConfigTrait.php b/lib/private/Files/ObjectStore/S3ConfigTrait.php index 121fa829d5484..2687fbd399309 100644 --- a/lib/private/Files/ObjectStore/S3ConfigTrait.php +++ b/lib/private/Files/ObjectStore/S3ConfigTrait.php @@ -31,6 +31,9 @@ trait S3ConfigTrait { protected string $bucket; + /** Maximum number of concurrent multipart uploads */ + protected int $concurrency; + protected int $timeout; protected string $proxy; diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index 2720902fa85f2..131014240f320 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -48,10 +48,10 @@ trait S3ConnectionTrait { protected string $id; - protected ?S3Client $connection; - protected bool $test; + protected ?S3Client $connection = null; + protected function parseParams($params) { if (empty($params['bucket'])) { throw new \Exception("Bucket has to be configured."); @@ -61,6 +61,8 @@ protected function parseParams($params) { $this->test = isset($params['test']); $this->bucket = $params['bucket']; + // Default to 5 like the S3 SDK does + $this->concurrency = $params['concurrency'] ?? 5; $this->proxy = $params['proxy'] ?? false; $this->timeout = $params['timeout'] ?? 15; $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD'; @@ -92,7 +94,7 @@ public function getProxy() { * @throws \Exception if connection could not be made */ public function getConnection() { - if (!is_null($this->connection)) { + if ($this->connection !== null) { return $this->connection; } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 3019805389d31..e0a94df1d9956 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -129,6 +129,7 @@ protected function writeSingle(string $urn, StreamInterface $stream, string $mim protected function writeMultiPart(string $urn, StreamInterface $stream, string $mimetype = null): void { $uploader = new MultipartUploader($this->getConnection(), $stream, [ 'bucket' => $this->bucket, + 'concurrency' => $this->concurrency, 'key' => $urn, 'part_size' => $this->uploadPartSize, 'params' => [