Handle quota properly and add part handling to storage

Signed-off-by: Julius Härtl <jus@bitgrid.net>

apps/dav/lib/DAV/CustomPropertiesBackend.php

@@ -180,7 +180,7 @@ public function propPatch($path, PropPatch $propPatch) {
 	 */
 	public function delete($path) {
 		$this->customPropertiesService->delete($this->user->getUID(), $path);
-		unset($this->cache[$path]);
+		unset($this->userCache[$path]);
 	}
 
 	/**

apps/dav/lib/Upload/ChunkingV2Plugin.php

@@ -29,7 +29,9 @@
 use OC\Files\View;
 use OCA\DAV\Connector\Sabre\Directory;
 use OCA\DAV\Connector\Sabre\FilesPlugin;
+use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
 use OCP\Files\Storage\IChunkedFileWrite;
+use OCP\Files\Storage\IProcessingCallbackStorage;
 use OCP\Files\Storage\IStorage;
 use OCP\Files\StorageInvalidException;
 use Sabre\DAV\Exception\BadRequest;
@@ -118,6 +120,10 @@ public function beforeMkcol(RequestInterface $request, ResponseInterface $respon
 
 	public function beforePut(RequestInterface $request, ResponseInterface $response): bool {
 		$this->uploadFolder = $this->server->tree->getNodeForPath(dirname($request->getPath()));
+		if (!$this->uploadFolder instanceof UploadFolder) {
+			return true;
+		}
+
 		try {
 			$this->checkPrerequisites();
 			$storage = $this->getStorage();
@@ -143,7 +149,7 @@ public function beforePut(RequestInterface $request, ResponseInterface $response
 
 		$additionalSize = (int)$request->getHeader('Content-Length');
 		if ($this->uploadFolder->childExists(self::TEMP_TARGET)) {
-			// FIXME Quota checking will not work for existing files that way
+			/** @var UploadFile $tempTargetFile */
 			$tempTargetFile = $this->uploadFolder->getChild(self::TEMP_TARGET);
 			$tempTargetCache = $storage->getCache()->get($tempTargetFile->getInternalPath());
 
@@ -159,7 +165,6 @@ public function beforePut(RequestInterface $request, ResponseInterface $response
 
 		$stream = $request->getBodyAsStream();
 		$storage->putChunkedFilePart($targetFile->getInternalPath(), $uploadId, (string)$partId, $stream, $additionalSize);
-		// FIXME add return value to putChunkedFilePart to validate against size
 
 		$storage->getCache()->update($cacheEntry->getId(), ['size' => $cacheEntry->getSize() + $additionalSize]);
 		if ($tempTargetFile) {
@@ -211,6 +216,22 @@ public function beforeMove($sourcePath, $destination): bool {
 
 		$rootView = new View();
 		if ($storage->instanceOfStorage(ObjectStoreStorage::class)) {
+			/** @var ObjectStoreStorage $storage */
+			$objectStore = $storage->getObjectStore();
+			if ($objectStore instanceof IObjectStoreMultiPartUpload) {
+				$parts = $objectStore->getMultipartUploads($storage->getURN($targetFile->getId()), $uploadId);
+				$size = 0;
+				foreach ($parts as $part) {
+					$size += $part['Size'];
+				}
+				$free = $storage->free_space($destinationParent->getInternalPath());
+				if ($free >= 0 && ($size > $free)) {
+					throw new InsufficientStorage("Insufficient space in $targetPath");
+				}
+			}
+		}
+		if ($storage->instanceOfStorage(IProcessingCallbackStorage::class)) {
+			/** @var IProcessingCallbackStorage $storage */
 			$lastTick = time();
 			$storage->processingCallback('writeChunkedFile', function () use ($lastTick) {
 				if ($lastTick < time()) {
@@ -245,7 +266,7 @@ public function beforeMove($sourcePath, $destination): bool {
 				['200' => [
 					FilesPlugin::SIZE_PROPERTYNAME => $rootView->filesize($destinationInView)
 				]],
-				$destinationExists ? 204 : 201
+				$destinationExists ? '204' : '201'
 			);
 			echo $this->server->xml->write(
 				'{DAV:}multistatus',

apps/dav/lib/Upload/UploadFile.php

@@ -45,6 +45,10 @@ public function get() {
 		return $this->file->get();
 	}
 
+	public function getId() {
+		return $this->file->getId();
+	}
+
 	public function getContentType() {
 		return $this->file->getContentType();
 	}

apps/dav/tests/unit/CapabilitiesTest.php

@@ -34,7 +34,7 @@ public function testGetCapabilities() {
 		$capabilities = new Capabilities();
 		$expected = [
 			'dav' => [
-				'chunking' => '1.0',
+				'chunking' => '2.0',
 			],
 		];
 		$this->assertSame($expected, $capabilities->getCapabilities());

build/psalm-baseline.xml

@@ -1014,6 +1014,12 @@
       <code>null</code>
     </NullArgument>
   </file>
+  <file src="apps/dav/lib/Upload/ChunkingV2Plugin.php">
+    <UndefinedFunction occurrences="2">
+      <code>Uri\split($destination)</code>
+      <code>Uri\split($targetPath)</code>
+    </UndefinedFunction>
+  </file>
   <file src="apps/dav/lib/Upload/UploadHome.php">
     <UndefinedFunction occurrences="1">
       <code>\Sabre\Uri\split($this-&gt;principalInfo['uri'])</code>

lib/composer/composer/autoload_classmap.php

@@ -322,6 +322,7 @@
     'OCP\\Files\\Storage\\IDisableEncryptionStorage' => $baseDir . '/lib/public/Files/Storage/IDisableEncryptionStorage.php',
     'OCP\\Files\\Storage\\ILockingStorage' => $baseDir . '/lib/public/Files/Storage/ILockingStorage.php',
     'OCP\\Files\\Storage\\INotifyStorage' => $baseDir . '/lib/public/Files/Storage/INotifyStorage.php',
+    'OCP\\Files\\Storage\\IProcessingCallbackStorage' => $baseDir . '/lib/public/Files/Storage/IProcessingCallbackStorage.php',
     'OCP\\Files\\Storage\\IStorage' => $baseDir . '/lib/public/Files/Storage/IStorage.php',
     'OCP\\Files\\Storage\\IStorageFactory' => $baseDir . '/lib/public/Files/Storage/IStorageFactory.php',
     'OCP\\Files\\Storage\\IWriteStreamStorage' => $baseDir . '/lib/public/Files/Storage/IWriteStreamStorage.php',

lib/composer/composer/autoload_static.php

@@ -351,6 +351,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
         'OCP\\Files\\Storage\\IDisableEncryptionStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IDisableEncryptionStorage.php',
         'OCP\\Files\\Storage\\ILockingStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/ILockingStorage.php',
         'OCP\\Files\\Storage\\INotifyStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/INotifyStorage.php',
+        'OCP\\Files\\Storage\\IProcessingCallbackStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IProcessingCallbackStorage.php',
         'OCP\\Files\\Storage\\IStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IStorage.php',
         'OCP\\Files\\Storage\\IStorageFactory' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IStorageFactory.php',
         'OCP\\Files\\Storage\\IWriteStreamStorage' => __DIR__ . '/../../..' . '/lib/public/Files/Storage/IWriteStreamStorage.php',

lib/private/Files/ObjectStore/ObjectStoreStorage.php

@@ -46,10 +46,11 @@
 use OCP\Files\ObjectStore\IObjectStore;
 use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
 use OCP\Files\Storage\IChunkedFileWrite;
+use OCP\Files\Storage\IProcessingCallbackStorage;
 use OCP\Files\Storage\IStorage;
 use OCP\ICache;
 
-class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite {
+class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite, IProcessingCallbackStorage {
 	use CopyDirectory;
 
 	/**
@@ -654,7 +655,7 @@ public function putChunkedFilePart(string $targetPath, string $writeToken, strin
 		$this->uploadCache->set($this->getUploadCacheKey($urn, $uploadId, 'parts'), $parts);
 	}
 
-	public function processingCallback(string $method, callable $callback) {
+	public function processingCallback(string $method, callable $callback): void {
 		if (in_array($method, ['writeChunkedFile'])) {
 			if (!isset($this->processingCallbacks[$method])) {
 				$this->processingCallbacks[$method] = [];

lib/private/Files/ObjectStore/S3.php

@@ -67,6 +67,17 @@ public function uploadMultipartPart(string $urn, string $uploadId, int $partId,
 		]);
 	}
 
+	public function getMultipartUploads(string $urn, string $uploadId): array {
+		$parts = $this->getConnection()->listParts([
+			'Bucket' => $this->bucket,
+			'Key' => $urn,
+			'UploadId' => $uploadId
+		]);
+		return array_map(function ($part) {
+			return $part;
+		}, $parts->get('Parts'));
+	}
+
 	public function completeMultipartUpload(string $urn, string $uploadId, array $result, callable $processingCallback = null): int {
 		$this->getConnection()->completeMultipartUpload([
 			'Bucket' => $this->bucket,

lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php

@@ -29,26 +29,31 @@
 use Aws\Result;
 
 /**
- * @since 22.0.0
+ * @since 23.0.0
  */
 interface IObjectStoreMultiPartUpload {
 	/**
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function initiateMultipartUpload(string $urn): string;
 
 	/**
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function uploadMultipartPart(string $urn, string $uploadId, int $partId, $stream, $size): Result;
 
 	/**
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function completeMultipartUpload(string $urn, string $uploadId, array $result): int;
 
 	/**
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function abortMultipartUpload(string $urn, string $uploadId): void;
+
+	/**
+	 * @since 23.0.0
+	 */
+	public function getMultipartUploads(string $urn, string $uploadId): array;
 }

lib/public/Files/Storage/IChunkedFileWrite.php

@@ -29,15 +29,15 @@
 use OCP\Files\GenericFileException;
 
 /**
- * @since 22.0.0
+ * @since 23.0.0
  */
 interface IChunkedFileWrite extends IStorage {
 
 	/**
 	 * @param string $targetPath Relative target path in the storage
 	 * @return string writeToken to be used with the other methods to uniquely identify the file write operation
 	 * @throws GenericFileException
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function beginChunkedFile(string $targetPath): string;
 
@@ -48,7 +48,7 @@ public function beginChunkedFile(string $targetPath): string;
 	 * @param resource $data
 	 * @param int|null $size
 	 * @throws GenericFileException
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function putChunkedFilePart(string $targetPath, string $writeToken, string $chunkId, $data, int $size = null): void;
 
@@ -57,15 +57,15 @@ public function putChunkedFilePart(string $targetPath, string $writeToken, strin
 	 * @param string $writeToken
 	 * @return int
 	 * @throws GenericFileException
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function writeChunkedFile(string $targetPath, string $writeToken): int;
 
 	/**
 	 * @param string $targetPath
 	 * @param string $writeToken
 	 * @throws GenericFileException
-	 * @since 22.0.0
+	 * @since 23.0.0
 	 */
 	public function cancelChunkedFile(string $targetPath, string $writeToken): void;
 }

lib/public/Files/Storage/IProcessingCallbackStorage.php

@@ -0,0 +1,46 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OCP\Files\Storage;
+
+use OCP\Files\GenericFileException;
+
+/**
+ * Interface that adds the ability to register processing callbacks for storage operation
+ *
+ * @since 23.0.0
+ */
+interface IProcessingCallbackStorage extends IStorage {
+	/**
+	 * Register a callback for a processing storage operation
+	 *
+	 * @param string $method
+	 * @param callable $callback being called regularly during the storage operation
+	 * @return void
+	 * @throws GenericFileException
+	 * @since 23.0.0
+	 */
+	public function processingCallback(string $method, callable $callback): void;
+}