S3 direct multipart upload optimisation.

The change was needed to avoid doubling of upload time as the original solution does a complete re-read of the S3 object on final MOVE of NextCloud chunked upload. See nextcloud#27034 for details.

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

apps/dav/composer/composer/autoload_classmap.php

@@ -271,6 +271,7 @@
     'OCA\\DAV\\Search\\EventsSearchProvider' => $baseDir . '/../lib/Search/EventsSearchProvider.php',
     'OCA\\DAV\\Search\\TasksSearchProvider' => $baseDir . '/../lib/Search/TasksSearchProvider.php',
     'OCA\\DAV\\Server' => $baseDir . '/../lib/Server.php',
+    'OCA\\DAV\\Service\\CustomPropertiesService' => $baseDir . '/../lib/Service/CustomPropertiesService.php',
     'OCA\\DAV\\Settings\\CalDAVSettings' => $baseDir . '/../lib/Settings/CalDAVSettings.php',
     'OCA\\DAV\\Storage\\PublicOwnerWrapper' => $baseDir . '/../lib/Storage/PublicOwnerWrapper.php',
     'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => $baseDir . '/../lib/SystemTag/SystemTagMappingNode.php',
@@ -283,6 +284,7 @@
     'OCA\\DAV\\Traits\\PrincipalProxyTrait' => $baseDir . '/../lib/Traits/PrincipalProxyTrait.php',
     'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
     'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
+    'OCA\\DAV\\Upload\\ChunkingV2Plugin' => $baseDir . '/../lib/Upload/ChunkingV2Plugin.php',
     'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
     'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
     'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',

apps/dav/composer/composer/autoload_static.php

@@ -286,6 +286,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Search\\EventsSearchProvider' => __DIR__ . '/..' . '/../lib/Search/EventsSearchProvider.php',
         'OCA\\DAV\\Search\\TasksSearchProvider' => __DIR__ . '/..' . '/../lib/Search/TasksSearchProvider.php',
         'OCA\\DAV\\Server' => __DIR__ . '/..' . '/../lib/Server.php',
+        'OCA\\DAV\\Service\\CustomPropertiesService' => __DIR__ . '/..' . '/../lib/Service/CustomPropertiesService.php',
         'OCA\\DAV\\Settings\\CalDAVSettings' => __DIR__ . '/..' . '/../lib/Settings/CalDAVSettings.php',
         'OCA\\DAV\\Storage\\PublicOwnerWrapper' => __DIR__ . '/..' . '/../lib/Storage/PublicOwnerWrapper.php',
         'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagMappingNode.php',
@@ -298,6 +299,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Traits\\PrincipalProxyTrait' => __DIR__ . '/..' . '/../lib/Traits/PrincipalProxyTrait.php',
         'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
         'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
+        'OCA\\DAV\\Upload\\ChunkingV2Plugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingV2Plugin.php',
         'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
         'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
         'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',

apps/dav/lib/BackgroundJob/UploadCleanup.php

@@ -28,6 +28,7 @@
 namespace OCA\DAV\BackgroundJob;
 
 use OC\User\NoUserException;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\TimedJob;
@@ -44,10 +45,14 @@ class UploadCleanup extends TimedJob {
 	/** @var IJobList */
 	private $jobList;
 
-	public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) {
+	/** @var CustomPropertiesService */
+	private $customPropertiesService;
+
+	public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList, CustomPropertiesService $customPropertiesService) {
 		parent::__construct($time);
 		$this->rootFolder = $rootFolder;
 		$this->jobList = $jobList;
+		$this->customPropertiesService = $customPropertiesService;
 
 		// Run once a day
 		$this->setInterval(60 * 60 * 24);
@@ -71,6 +76,8 @@ protected function run($argument) {
 
 		$files = $uploadFolder->getDirectoryListing();
 
+		$davPath = 'uploads/' . $uid . '/' . $uploadFolder->getName();
+
 		// Remove if all files have an mtime of more than a day
 		$time = $this->time->getTime() - 60 * 60 * 24;
 
@@ -82,6 +89,7 @@ protected function run($argument) {
 		}, $initial);
 
 		if ($expire) {
+			$this->customPropertiesService->delete($uid, $davPath);
 			$uploadFolder->delete();
 			$this->jobList->remove(self::class, $argument);
 		}

apps/dav/lib/Connector/Sabre/Directory.php

@@ -37,6 +37,7 @@
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
+use OCA\DAV\Upload\FutureFile;
 use OCP\Files\FileInfo;
 use OCP\Files\ForbiddenException;
 use OCP\Files\InvalidPathException;

apps/dav/lib/Connector/Sabre/Node.php

@@ -246,6 +246,14 @@ public function getInternalFileId() {
 		return $this->info->getId();
 	}
 
+	public function getInternalPath(): string {
+		return $this->info->getInternalPath();
+	}
+
+	public function getAbsoluteInternalPath(): string {
+		return $this->info->getPath();
+	}
+
 	/**
 	 * @param string $user
 	 * @return int

apps/dav/lib/Connector/Sabre/ServerFactory.php

@@ -34,6 +34,7 @@
 use OC\Files\Node\Folder;
 use OCA\DAV\AppInfo\PluginManager;
 use OCA\DAV\Files\BrowserErrorPagePlugin;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\Files\Mount\IMountManager;
 use OCP\IConfig;
 use OCP\IDBConnection;
@@ -204,6 +205,7 @@ public function createServer($baseUri,
 						new \OCA\DAV\DAV\CustomPropertiesBackend(
 							$objectTree,
 							$this->databaseConnection,
+							\OC::$server->get(CustomPropertiesService::class),
 							$this->userSession->getUser()
 						)
 					)

apps/dav/lib/DAV/CustomPropertiesBackend.php

@@ -25,6 +25,7 @@
 namespace OCA\DAV\DAV;
 
 use OCA\DAV\Connector\Sabre\Node;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\IDBConnection;
 use OCP\IUser;
 use Sabre\DAV\PropertyStorage\Backend\BackendInterface;
@@ -75,6 +76,11 @@ class CustomPropertiesBackend implements BackendInterface {
 	 */
 	private $connection;
 
+	/**
+	 * @var CustomPropertiesService
+	 */
+	private $customPropertiesService;
+
 	/**
 	 * @var IUser
 	 */
@@ -95,9 +101,11 @@ class CustomPropertiesBackend implements BackendInterface {
 	public function __construct(
 		Tree $tree,
 		IDBConnection $connection,
+		CustomPropertiesService $customPropertiesService,
 		IUser $user) {
 		$this->tree = $tree;
 		$this->connection = $connection;
+		$this->customPropertiesService = $customPropertiesService;
 		$this->user = $user;
 	}
 
@@ -171,13 +179,8 @@ public function propPatch($path, PropPatch $propPatch) {
 	 * @param string $path path of node for which to delete properties
 	 */
 	public function delete($path) {
-		$statement = $this->connection->prepare(
-			'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
-		);
-		$statement->execute([$this->user->getUID(), $this->formatPath($path)]);
-		$statement->closeCursor();
-
-		unset($this->userCache[$path]);
+		$this->customPropertiesService->delete($this->user->getUID(), $path);
+		unset($this->cache[$path]);
 	}
 
 	/**

apps/dav/lib/Server.php

@@ -63,8 +63,10 @@
 use OCA\DAV\Files\BrowserErrorPagePlugin;
 use OCA\DAV\Files\LazySearchBackend;
 use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCA\DAV\SystemTag\SystemTagPlugin;
 use OCA\DAV\Upload\ChunkingPlugin;
+use OCA\DAV\Upload\ChunkingV2Plugin;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\IRequest;
 use OCP\SabrePluginEvent;
@@ -203,6 +205,7 @@ public function __construct(IRequest $request, $baseUri) {
 		));
 
 		$this->server->addPlugin(new CopyEtagHeaderPlugin());
+		$this->server->addPlugin(new ChunkingV2Plugin());
 		$this->server->addPlugin(new ChunkingPlugin());
 
 		// allow setup of additional plugins
@@ -249,6 +252,7 @@ public function __construct(IRequest $request, $baseUri) {
 						new CustomPropertiesBackend(
 							$this->server->tree,
 							\OC::$server->getDatabaseConnection(),
+							\OC::$server->get(CustomPropertiesService::class),
 							\OC::$server->getUserSession()->getUser()
 						)
 					)

apps/dav/lib/Service/CustomPropertiesService.php

@@ -0,0 +1,60 @@
+<?php
+/*
+ * @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+declare(strict_types=1);
+
+
+namespace OCA\DAV\Service;
+
+use OCP\IDBConnection;
+
+class CustomPropertiesService {
+
+	/** @var IDBConnection */
+	private $connection;
+
+	public function __construct(IDBConnection $connection) {
+		$this->connection = $connection;
+	}
+
+	public function delete(string $userId, string $path): void {
+		$statement = $this->connection->prepare(
+			'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
+		);
+		$result = $statement->execute([$userId, $this->formatPath($path)]);
+		$result->closeCursor();
+	}
+
+	/**
+	 * long paths are hashed to ensure they fit in the database
+	 *
+	 * @param string $path
+	 * @return string
+	 */
+	private function formatPath(string $path): string {
+		if (strlen($path) > 250) {
+			return sha1($path);
+		}
+		return $path;
+	}
+}