FilesMetadata

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

core/Command/FilesMetadata/Get.php

@@ -0,0 +1,55 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\Core\Command\FilesMetadata;
+
+use OCP\FilesMetadata\IFilesMetadataManager;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Get extends Command {
+	public function __construct(
+		private IFilesMetadataManager $filesMetadataManager,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this->setName('metadata:get')
+			 ->setDescription('update and returns up-to-date metadata')
+			 ->addArgument(
+				 'fileId',
+				 InputArgument::REQUIRED,
+				 'id of the file document'
+			 )
+			 ->addOption(
+				 'background',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'emulate background jobs env'
+			 )
+			 ->addOption(
+				 'refresh',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'refresh metadata'
+			 );
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$fileId = (int) $input->getArgument('fileId');
+		if ($input->getOption('refresh')) {
+			$metadata = $this->filesMetadataManager->refreshMetadata($fileId, $input->getOption('background'));
+		} else {
+			$metadata = $this->filesMetadataManager->getMetadata($fileId);
+		}
+
+		$output->writeln(json_encode($metadata, JSON_PRETTY_PRINT));
+
+		return 0;
+	}
+}

core/Migrations/Version28000Date20231004103301.php

@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version28000Date20231004103301 extends SimpleMigrationStep {
+
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('files_metadata')) {
+			$table = $schema->createTable('files_metadata');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, [
+				'notnull' => false,
+				'length' => 15,
+			]);
+			$table->addColumn('json', Types::TEXT);
+			$table->addColumn('sync_token', Types::STRING, [
+				'length' => 15,
+			]);
+			$table->addColumn('last_update', Types::DATETIME);
+
+			$table->setPrimaryKey(['id']);
+			$table->addUniqueIndex(['file_id'], 'files_meta_fileid');
+		}
+
+		if (!$schema->hasTable('files_metadata_index')) {
+			$table = $schema->createTable('files_metadata_index');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, [
+				'notnull' => false,
+				'length' => 15,
+			]);
+			$table->addColumn('meta_key', Types::STRING, [
+				'notnull' => false,
+				'length' => 31,
+			]);
+			$table->addColumn('meta_value', Types::STRING, [
+				'notnull' => false,
+				'length' => 63,
+			]);
+			$table->addColumn('meta_value_int', Types::BIGINT, [
+				'notnull' => false,
+				'length' => 11,
+			]);
+
+			$table->setPrimaryKey(['id']);
+			$table->addIndex(['meta_key', 'meta_value'], 'f_meta_index');
+			$table->addIndex(['meta_key', 'meta_value_int'], 'f_meta_index_i');
+		}
+
+		return $schema;
+	}
+}

core/register_command.php

@@ -214,6 +214,8 @@
 	$application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager()));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class));
+
+	$application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class));
 } else {
 	$application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class));
 }

lib/private/FilesMetadata/Event/FilesMetadataBackgroundEvent.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\FilesMetadata\Event;
+
+use OCP\EventDispatcher\Event;
+use OCP\FilesMetadata\IFilesMetadataEvent;
+use OCP\FilesMetadata\Model\IFilesMetadata;
+
+class FilesMetadataBackgroundEvent extends Event implements IFilesMetadataEvent {
+	public function __construct(
+		private int $fileId,
+		private IFilesMetadata $metadata,
+	) {
+		parent::__construct();
+	}
+
+	public function getFileId(): int {
+		return $this->fileId;
+	}
+
+	public function getMetadata(): IFilesMetadata {
+		return $this->metadata;
+	}
+}

lib/private/FilesMetadata/Event/FilesMetadataEvent.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\FilesMetadata\Event;
+
+use OCP\EventDispatcher\Event;
+use OCP\FilesMetadata\IFilesMetadataEvent;
+use OCP\FilesMetadata\Model\IFilesMetadata;
+
+class FilesMetadataEvent extends Event implements IFilesMetadataEvent {
+
+	public function __construct(
+		private int $fileId,
+		private IFilesMetadata $metadata,
+		private bool $runAsBackgroundJob = false
+	) {
+		parent::__construct();
+	}
+
+
+	/**
+	 * If an app prefer to update metadata on a background job, instead of
+	 * live process, just call this method.
+	 * A new event will be generated on next cron tick
+	 *
+	 * @return void
+	 */
+	public function requestBackgroundJob(): void {
+		$this->runAsBackgroundJob = true;
+	}
+
+	/**
+	 * return fileId
+	 *
+	 * @return int
+	 */
+	public function getFileId(): int {
+		return $this->fileId;
+	}
+
+	/**
+	 * return Metadata
+	 *
+	 * @return IFilesMetadata
+	 */
+	public function getMetadata(): IFilesMetadata {
+		return $this->metadata;
+	}
+
+	/**
+	 * return true if any app that catch this event requested a re-run as background job
+	 *
+	 * @return bool
+	 */
+	public function isRunAsBackgroundJobRequested(): bool {
+		return $this->runAsBackgroundJob;
+	}
+}

lib/private/FilesMetadata/FilesMetadataManager.php

@@ -0,0 +1,133 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\FilesMetadata;
+
+use OC\FilesMetadata\Event\FilesMetadataEvent;
+use OC\FilesMetadata\Listener\FilesMetadataDelete;
+use OC\FilesMetadata\Listener\FilesMetadataUpdate;
+use OC\FilesMetadata\Model\FilesMetadata;
+use OC\FilesMetadata\Service\MetadataRequestService;
+use OCP\DB\Exception;
+use OCP\EventDispatcher\IEventDispatcher;
+use OCP\Files\Events\Node\NodeCreatedEvent;
+use OCP\Files\Events\Node\NodeDeletedEvent;
+use OCP\Files\Events\Node\NodeWrittenEvent;
+use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
+use OCP\FilesMetadata\IFilesMetadataManager;
+use OCP\FilesMetadata\IFilesMetadataQueryHelper;
+use OCP\FilesMetadata\Model\IFilesMetadata;
+use Psr\Log\LoggerInterface;
+
+class FilesMetadataManager implements IFilesMetadataManager {
+	public function __construct(
+		private IEventDispatcher $eventDispatcher,
+		private FilesMetadataQueryHelper $filesMetadataQueryHelper,
+		private MetadataRequestService $metadataRequestService,
+		private LoggerInterface $logger
+	) {
+	}
+
+	/**
+	 * @param int $fileId
+	 * @param bool $createIfNeeded
+	 *
+	 * @return IFilesMetadata
+	 * @throws FilesMetadataNotFoundException
+	 */
+	public function getMetadata(int $fileId, bool $createIfNeeded = false): IFilesMetadata {
+		try {
+			return $this->metadataRequestService->getMetadataFromFileId($fileId);
+		} catch (FilesMetadataNotFoundException $e) {
+			if ($createIfNeeded) {
+				return $this->refreshMetadata($fileId);
+			}
+
+			throw $e;
+		}
+	}
+
+	public function refreshMetadata(
+		int $fileId,
+		bool $asBackgroundJob = false,
+		bool $fromScratch = false
+	): IFilesMetadata {
+		$metadata = null;
+		if (!$fromScratch) {
+			try {
+				$metadata = $this->metadataRequestService->getMetadataFromFileId($fileId);
+			} catch (FilesMetadataNotFoundException $e) {
+			}
+		}
+
+		if (is_null($metadata)) {
+			$metadata = new FilesMetadata($fileId);
+		}
+
+		$event = new FilesMetadataEvent($fileId, $metadata, $asBackgroundJob);
+		$this->eventDispatcher->dispatchTyped($event);
+		$this->saveMetadata($event->getMetadata());
+
+		if (!$asBackgroundJob && $event->isRunAsBackgroundJobRequested()) {
+			// create entry in jobs
+			//https://github.com/nextcloud/photos/blob/6b01dce7cd13bc5fb319bd114d22bc31f295a245/lib/Listener/PlaceManagerEventListener.php#L63-L64
+			//https://github.com/nextcloud/photos/blob/6b01dce7cd13bc5fb319bd114d22bc31f295a245/lib/Jobs/MapMediaToPlaceJob.php#L46
+		}
+		return $metadata;
+	}
+
+	/**
+	 * @param IFilesMetadata $filesMetadata
+	 *
+	 * @return void
+	 */
+	public function saveMetadata(IFilesMetadata $filesMetadata): void {
+		if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) {
+			return;
+		}
+
+		try {
+			// if update request changed no rows, means that new entry is needed, or sync_token not valid anymore
+			$updated = $this->metadataRequestService->updateMetadata($filesMetadata);
+			if ($updated === 0) {
+				$this->metadataRequestService->store($filesMetadata);
+			}
+		} catch (\OCP\DB\Exception $e) {
+			// if duplicate, only means a desync during update. cancel update process.
+			if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
+				$this->logger->warning('issue while saveMetadata', ['exception' => $e, 'metadata' => $filesMetadata]);
+			}
+
+			return;
+		}
+
+//		$this->removeDeprecatedMetadata($filesMetadata);
+		// remove indexes from metadata_index that are not in the list of indexes anymore.
+		foreach ($filesMetadata->getIndexes() as $index) {
+			$this->metadataRequestService->updateIndex($filesMetadata, $index);
+			// foreach index, update entry in table metadata_index
+			// if no update, insert as new row
+			// !! we might want to get the type of the value to be indexed at one point !!
+		}
+	}
+
+
+	public function deleteMetadata(int $fileId): void {
+		try {
+			$this->metadataRequestService->dropMetadata($fileId);
+		} catch (Exception $e) {
+			$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]);
+		}
+	}
+
+	public function getQueryHelper(): IFilesMetadataQueryHelper {
+		return $this->filesMetadataQueryHelper;
+	}
+
+	public static function loadListeners(IEventDispatcher $eventDispatcher): void {
+		$eventDispatcher->addServiceListener(NodeCreatedEvent::class, FilesMetadataUpdate::class);
+		$eventDispatcher->addServiceListener(NodeWrittenEvent::class, FilesMetadataUpdate::class);
+		$eventDispatcher->addServiceListener(NodeDeletedEvent::class, FilesMetadataDelete::class);
+	}
+}