From d9b37ac5fa11afd519fadf3bb2f9a81f74cf7e4d Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 17 Jun 2024 17:10:25 +0200 Subject: [PATCH 1/2] fix: move repair mimetype repair step to the expensive steps Signed-off-by: Robin Appelman --- lib/private/Repair.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/Repair.php b/lib/private/Repair.php index b265d5767eabc..04de9c5702df7 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -147,7 +147,6 @@ public function addStep($repairStep) { public static function getRepairSteps(): array { return [ new Collation(\OC::$server->getConfig(), \OC::$server->get(LoggerInterface::class), \OC::$server->getDatabaseConnection(), false), - new RepairMimeTypes(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()), new CleanTags(\OC::$server->getDatabaseConnection(), \OC::$server->getUserManager()), new RepairInvalidShares(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()), new MoveUpdaterStepFile(\OC::$server->getConfig()), @@ -198,6 +197,7 @@ public static function getRepairSteps(): array { public static function getExpensiveRepairSteps() { return [ new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()), + new RepairMimeTypes(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection()), \OC::$server->get(ValidatePhoneNumber::class), \OC::$server->get(DeleteSchedulingObjects::class), ]; From e74f71b32e0943bacd3af5e2d37dd794fa7058e4 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 18 Jun 2024 11:27:29 +0200 Subject: [PATCH 2/2] feat: add setup check for needed mimetype migrations Signed-off-by: Robin Appelman --- .../composer/composer/autoload_classmap.php | 1 + .../composer/composer/autoload_static.php | 1 + apps/settings/lib/AppInfo/Application.php | 2 + .../MimeTypeMigrationAvailable.php | 44 +++++++++++ lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + lib/private/Migration/NullOutput.php | 35 +++++++++ lib/private/Repair/RepairMimeTypes.php | 78 ++++++++++++------- tests/lib/Repair/RepairMimeTypesTest.php | 6 +- 9 files changed, 140 insertions(+), 29 deletions(-) create mode 100644 apps/settings/lib/SetupChecks/MimeTypeMigrationAvailable.php create mode 100644 lib/private/Migration/NullOutput.php diff --git a/apps/settings/composer/composer/autoload_classmap.php b/apps/settings/composer/composer/autoload_classmap.php index 6bbb2978a875e..488aaee264d2a 100644 --- a/apps/settings/composer/composer/autoload_classmap.php +++ b/apps/settings/composer/composer/autoload_classmap.php @@ -102,6 +102,7 @@ 'OCA\\Settings\\SetupChecks\\LegacySSEKeyFormat' => $baseDir . '/../lib/SetupChecks/LegacySSEKeyFormat.php', 'OCA\\Settings\\SetupChecks\\MaintenanceWindowStart' => $baseDir . '/../lib/SetupChecks/MaintenanceWindowStart.php', 'OCA\\Settings\\SetupChecks\\MemcacheConfigured' => $baseDir . '/../lib/SetupChecks/MemcacheConfigured.php', + 'OCA\\Settings\\SetupChecks\\MimeTypeMigrationAvailable' => $baseDir . '/../lib/SetupChecks/MimeTypeMigrationAvailable.php', 'OCA\\Settings\\SetupChecks\\MysqlUnicodeSupport' => $baseDir . '/../lib/SetupChecks/MysqlUnicodeSupport.php', 'OCA\\Settings\\SetupChecks\\OcxProviders' => $baseDir . '/../lib/SetupChecks/OcxProviders.php', 'OCA\\Settings\\SetupChecks\\OverwriteCliUrl' => $baseDir . '/../lib/SetupChecks/OverwriteCliUrl.php', diff --git a/apps/settings/composer/composer/autoload_static.php b/apps/settings/composer/composer/autoload_static.php index df8f985d7ab60..ac2e464523986 100644 --- a/apps/settings/composer/composer/autoload_static.php +++ b/apps/settings/composer/composer/autoload_static.php @@ -117,6 +117,7 @@ class ComposerStaticInitSettings 'OCA\\Settings\\SetupChecks\\LegacySSEKeyFormat' => __DIR__ . '/..' . '/../lib/SetupChecks/LegacySSEKeyFormat.php', 'OCA\\Settings\\SetupChecks\\MaintenanceWindowStart' => __DIR__ . '/..' . '/../lib/SetupChecks/MaintenanceWindowStart.php', 'OCA\\Settings\\SetupChecks\\MemcacheConfigured' => __DIR__ . '/..' . '/../lib/SetupChecks/MemcacheConfigured.php', + 'OCA\\Settings\\SetupChecks\\MimeTypeMigrationAvailable' => __DIR__ . '/..' . '/../lib/SetupChecks/MimeTypeMigrationAvailable.php', 'OCA\\Settings\\SetupChecks\\MysqlUnicodeSupport' => __DIR__ . '/..' . '/../lib/SetupChecks/MysqlUnicodeSupport.php', 'OCA\\Settings\\SetupChecks\\OcxProviders' => __DIR__ . '/..' . '/../lib/SetupChecks/OcxProviders.php', 'OCA\\Settings\\SetupChecks\\OverwriteCliUrl' => __DIR__ . '/..' . '/../lib/SetupChecks/OverwriteCliUrl.php', diff --git a/apps/settings/lib/AppInfo/Application.php b/apps/settings/lib/AppInfo/Application.php index 2a6b21f7f7d34..f62555f0cdb18 100644 --- a/apps/settings/lib/AppInfo/Application.php +++ b/apps/settings/lib/AppInfo/Application.php @@ -45,6 +45,7 @@ use OCA\Settings\SetupChecks\LegacySSEKeyFormat; use OCA\Settings\SetupChecks\MaintenanceWindowStart; use OCA\Settings\SetupChecks\MemcacheConfigured; +use OCA\Settings\SetupChecks\MimeTypeMigrationAvailable; use OCA\Settings\SetupChecks\MysqlUnicodeSupport; use OCA\Settings\SetupChecks\OcxProviders; use OCA\Settings\SetupChecks\OverwriteCliUrl; @@ -176,6 +177,7 @@ public function register(IRegistrationContext $context): void { $context->registerSetupCheck(LegacySSEKeyFormat::class); $context->registerSetupCheck(MaintenanceWindowStart::class); $context->registerSetupCheck(MemcacheConfigured::class); + $context->registerSetupCheck(MimeTypeMigrationAvailable::class); $context->registerSetupCheck(MysqlUnicodeSupport::class); $context->registerSetupCheck(OcxProviders::class); $context->registerSetupCheck(OverwriteCliUrl::class); diff --git a/apps/settings/lib/SetupChecks/MimeTypeMigrationAvailable.php b/apps/settings/lib/SetupChecks/MimeTypeMigrationAvailable.php new file mode 100644 index 0000000000000..98c8eacbfdaa1 --- /dev/null +++ b/apps/settings/lib/SetupChecks/MimeTypeMigrationAvailable.php @@ -0,0 +1,44 @@ +l10n = $l10nFactory->get('core'); + } + + public function getCategory(): string { + return 'system'; + } + + public function getName(): string { + return $this->l10n->t('Mimetype migrations available'); + } + + public function run(): SetupResult { + if ($this->repairMimeTypes->migrationsAvailable()) { + return SetupResult::warning( + $this->l10n->t('One or more mimetype migrations are available. Occasionally new mimetypes are added to better handle certain file types. Migrating the mimetypes take a long time on larger instances so this is not done automatically during upgrades. Use the command `occ maintenance:repair --include-expensive` to perform the migrations.'), + ); + } else { + return SetupResult::success('None'); + } + } +} diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index dbd9ebc66ab58..0837fe2ab62b1 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1607,6 +1607,7 @@ 'OC\\MemoryInfo' => $baseDir . '/lib/private/MemoryInfo.php', 'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php', + 'OC\\Migration\\NullOutput' => $baseDir . '/lib/private/Migration/NullOutput.php', 'OC\\Migration\\SimpleOutput' => $baseDir . '/lib/private/Migration/SimpleOutput.php', 'OC\\NaturalSort' => $baseDir . '/lib/private/NaturalSort.php', 'OC\\NaturalSort_DefaultCollator' => $baseDir . '/lib/private/NaturalSort_DefaultCollator.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index f58c17bf411ee..b7f11b43266df 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1640,6 +1640,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\MemoryInfo' => __DIR__ . '/../../..' . '/lib/private/MemoryInfo.php', 'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php', + 'OC\\Migration\\NullOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/NullOutput.php', 'OC\\Migration\\SimpleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/SimpleOutput.php', 'OC\\NaturalSort' => __DIR__ . '/../../..' . '/lib/private/NaturalSort.php', 'OC\\NaturalSort_DefaultCollator' => __DIR__ . '/../../..' . '/lib/private/NaturalSort_DefaultCollator.php', diff --git a/lib/private/Migration/NullOutput.php b/lib/private/Migration/NullOutput.php new file mode 100644 index 0000000000000..3f4cc38dba8ef --- /dev/null +++ b/lib/private/Migration/NullOutput.php @@ -0,0 +1,35 @@ +config = $config; - $this->connection = $connection; + public function __construct( + protected IConfig $config, + protected IDBConnection $connection + ) { } public function getName() { @@ -33,6 +31,10 @@ public function getName() { } private function updateMimetypes($updatedMimetypes) { + if ($this->dryRun) { + $this->changeCount += count($updatedMimetypes); + return; + } $query = $this->connection->getQueryBuilder(); $query->select('id') ->from('mimetypes') @@ -236,77 +238,99 @@ private function introduceReStructuredTextFormatType() { return $this->updateMimetypes($updatedMimetypes); } + public function migrationsAvailable(): bool { + $this->dryRun = true; + $this->run(new NullOutput()); + $this->dryRun = false; + return $this->changeCount > 0; + } + + private function getMimeTypeVersion(): string { + $serverVersion = $this->config->getSystemValueString('version', '0.0.0'); + // 29.0.0.10 is the last version with a mimetype migration before it was moved to a separate version number + if (version_compare($serverVersion, '29.0.0.10', '>')) { + return $this->config->getAppValue('files', 'mimetype_version', '29.0.0.10'); + } else { + return $serverVersion; + } + } + /** * Fix mime types */ public function run(IOutput $out) { - $ocVersionFromBeforeUpdate = $this->config->getSystemValueString('version', '0.0.0'); + $serverVersion = $this->config->getSystemValueString('version', '0.0.0'); + $mimeTypeVersion = $this->getMimeTypeVersion(); // NOTE TO DEVELOPERS: when adding new mime types, please make sure to // add a version comparison to avoid doing it every time - if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.14', '<') && $this->introduceImageTypes()) { + if (version_compare($mimeTypeVersion, '12.0.0.14', '<') && $this->introduceImageTypes()) { $out->info('Fixed image mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '12.0.0.13', '<') && $this->introduceWindowsProgramTypes()) { + if (version_compare($mimeTypeVersion, '12.0.0.13', '<') && $this->introduceWindowsProgramTypes()) { $out->info('Fixed windows program mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.0', '<') && $this->introduceLocationTypes()) { + if (version_compare($mimeTypeVersion, '13.0.0.0', '<') && $this->introduceLocationTypes()) { $out->info('Fixed geospatial mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.3', '<') && $this->introduceInternetShortcutTypes()) { + if (version_compare($mimeTypeVersion, '13.0.0.3', '<') && $this->introduceInternetShortcutTypes()) { $out->info('Fixed internet-shortcut mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '13.0.0.6', '<') && $this->introduceStreamingTypes()) { + if (version_compare($mimeTypeVersion, '13.0.0.6', '<') && $this->introduceStreamingTypes()) { $out->info('Fixed streaming mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.8', '<') && $this->introduceVisioTypes()) { + if (version_compare($mimeTypeVersion, '14.0.0.8', '<') && $this->introduceVisioTypes()) { $out->info('Fixed visio mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '14.0.0.10', '<') && $this->introduceComicbookTypes()) { + if (version_compare($mimeTypeVersion, '14.0.0.10', '<') && $this->introduceComicbookTypes()) { $out->info('Fixed comicbook mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '20.0.0.5', '<') && $this->introduceOpenDocumentTemplates()) { + if (version_compare($mimeTypeVersion, '20.0.0.5', '<') && $this->introduceOpenDocumentTemplates()) { $out->info('Fixed OpenDocument template mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '21.0.0.7', '<') && $this->introduceOrgModeType()) { + if (version_compare($mimeTypeVersion, '21.0.0.7', '<') && $this->introduceOrgModeType()) { $out->info('Fixed orgmode mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '23.0.0.2', '<') && $this->introduceFlatOpenDocumentType()) { + if (version_compare($mimeTypeVersion, '23.0.0.2', '<') && $this->introduceFlatOpenDocumentType()) { $out->info('Fixed Flat OpenDocument mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '25.0.0.2', '<') && $this->introduceOnlyofficeFormType()) { + if (version_compare($mimeTypeVersion, '25.0.0.2', '<') && $this->introduceOnlyofficeFormType()) { $out->info('Fixed ONLYOFFICE Forms OpenXML mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '26.0.0.1', '<') && $this->introduceAsciidocType()) { + if (version_compare($mimeTypeVersion, '26.0.0.1', '<') && $this->introduceAsciidocType()) { $out->info('Fixed AsciiDoc mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '28.0.0.5', '<') && $this->introduceEnhancedMetafileFormatType()) { + if (version_compare($mimeTypeVersion, '28.0.0.5', '<') && $this->introduceEnhancedMetafileFormatType()) { $out->info('Fixed Enhanced Metafile Format mime types'); } - if (version_compare($ocVersionFromBeforeUpdate, '29.0.0.2', '<') && $this->introduceEmlAndMsgFormatType()) { + if (version_compare($mimeTypeVersion, '29.0.0.2', '<') && $this->introduceEmlAndMsgFormatType()) { $out->info('Fixed eml and msg mime type'); } - if (version_compare($ocVersionFromBeforeUpdate, '29.0.0.6', '<') && $this->introduceAacAudioType()) { + if (version_compare($mimeTypeVersion, '29.0.0.6', '<') && $this->introduceAacAudioType()) { $out->info('Fixed aac mime type'); } - if (version_compare($ocVersionFromBeforeUpdate, '29.0.0.10', '<') && $this->introduceReStructuredTextFormatType()) { + if (version_compare($mimeTypeVersion, '29.0.0.10', '<') && $this->introduceReStructuredTextFormatType()) { $out->info('Fixed ReStructured Text mime type'); } + + if (!$this->dryRun) { + $this->config->setAppValue('files', 'mimetype_version', $serverVersion); + } } } diff --git a/tests/lib/Repair/RepairMimeTypesTest.php b/tests/lib/Repair/RepairMimeTypesTest.php index b7076b32e77ec..fbccd393fd0a4 100644 --- a/tests/lib/Repair/RepairMimeTypesTest.php +++ b/tests/lib/Repair/RepairMimeTypesTest.php @@ -39,10 +39,12 @@ protected function setUp(): void { $config = $this->getMockBuilder(IConfig::class) ->disableOriginalConstructor() ->getMock(); - $config->expects($this->any()) - ->method('getSystemValueString') + $config->method('getSystemValueString') ->with('version') ->willReturn('11.0.0.0'); + $config->method('getAppValue') + ->with('files', 'mimetype_version') + ->willReturn('11.0.0.0'); $this->storage = new \OC\Files\Storage\Temporary([]);