From 17e48ee1a724e23189e9781b81262ff21bb82361 Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Wed, 28 Aug 2024 11:50:23 +0200 Subject: [PATCH 01/12] feat(textprocessing): TextProcessingManager::runTask calls TaskProcessingManager::runTask Signed-off-by: Julien Veyssier --- lib/private/TaskProcessing/Manager.php | 31 ++++++++++++-- lib/private/TextProcessing/Manager.php | 56 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/lib/private/TaskProcessing/Manager.php b/lib/private/TaskProcessing/Manager.php index 2730c777cc980..3713d5915754d 100644 --- a/lib/private/TaskProcessing/Manager.php +++ b/lib/private/TaskProcessing/Manager.php @@ -87,7 +87,6 @@ public function __construct( private IEventDispatcher $dispatcher, IAppDataFactory $appDataFactory, private IRootFolder $rootFolder, - private \OCP\TextProcessing\IManager $textProcessingManager, private \OCP\TextToImage\IManager $textToImageManager, private \OCP\SpeechToText\ISpeechToTextManager $speechToTextManager, private IUserMountCache $userMountCache, @@ -98,8 +97,34 @@ public function __construct( } + /** + * This is almost a copy of textProcessingManager->getProviders + * to avoid a dependency cycle between TextProcessingManager and TaskProcessingManager + */ + private function _getRawTextProcessingProviders(): array { + $context = $this->coordinator->getRegistrationContext(); + if ($context === null) { + return []; + } + + $providers = []; + + foreach ($context->getTextProcessingProviders() as $providerServiceRegistration) { + $class = $providerServiceRegistration->getService(); + try { + $providers[$class] = $this->serverContainer->get($class); + } catch (\Throwable $e) { + $this->logger->error('Failed to load Text processing provider ' . $class, [ + 'exception' => $e, + ]); + } + } + + return $providers; + } + private function _getTextProcessingProviders(): array { - $oldProviders = $this->textProcessingManager->getProviders(); + $oldProviders = $this->_getRawTextProcessingProviders(); $newProviders = []; foreach ($oldProviders as $oldProvider) { $provider = new class($oldProvider) implements IProvider, ISynchronousProvider { @@ -190,7 +215,7 @@ public function getOptionalOutputShapeEnumValues(): array { * @return ITaskType[] */ private function _getTextProcessingTaskTypes(): array { - $oldProviders = $this->textProcessingManager->getProviders(); + $oldProviders = $this->_getRawTextProcessingProviders(); $newTaskTypes = []; foreach ($oldProviders as $oldProvider) { // These are already implemented in the TaskProcessing realm diff --git a/lib/private/TextProcessing/Manager.php b/lib/private/TextProcessing/Manager.php index a03c028a5c922..b9e89e1211b35 100644 --- a/lib/private/TextProcessing/Manager.php +++ b/lib/private/TextProcessing/Manager.php @@ -20,13 +20,22 @@ use OCP\IConfig; use OCP\IServerContainer; use OCP\PreConditionNotMetException; +use OCP\TaskProcessing\IManager as TaskProcessingIManager; +use OCP\TaskProcessing\TaskTypes\TextToText; +use OCP\TaskProcessing\TaskTypes\TextToTextHeadline; +use OCP\TaskProcessing\TaskTypes\TextToTextSummary; +use OCP\TaskProcessing\TaskTypes\TextToTextTopics; use OCP\TextProcessing\Exception\TaskFailureException; +use OCP\TextProcessing\FreePromptTaskType; +use OCP\TextProcessing\HeadlineTaskType; use OCP\TextProcessing\IManager; use OCP\TextProcessing\IProvider; use OCP\TextProcessing\IProviderWithExpectedRuntime; use OCP\TextProcessing\IProviderWithId; +use OCP\TextProcessing\SummaryTaskType; use OCP\TextProcessing\Task; use OCP\TextProcessing\Task as OCPTask; +use OCP\TextProcessing\TopicsTaskType; use Psr\Log\LoggerInterface; use RuntimeException; use Throwable; @@ -42,6 +51,7 @@ public function __construct( private IJobList $jobList, private TaskMapper $taskMapper, private IConfig $config, + private TaskProcessingIManager $taskProcessingManager, ) { } @@ -98,6 +108,52 @@ public function canHandleTask(OCPTask $task): bool { * @inheritDoc */ public function runTask(OCPTask $task): string { + // try to run a task processing task if possible + $taskTypeClass = $task->getType(); + $taskProcessingCompatibleTaskTypes = [ + FreePromptTaskType::class => TextToText::ID, + HeadlineTaskType::class => TextToTextHeadline::ID, + SummaryTaskType::class => TextToTextSummary::ID, + TopicsTaskType::class => TextToTextTopics::ID, + ]; + if (isset($taskProcessingCompatibleTaskTypes[$taskTypeClass])) { + try { + $taskProcessingTaskTypeId = $taskProcessingCompatibleTaskTypes[$taskTypeClass]; + $taskProcessingTask = new \OCP\TaskProcessing\Task( + $taskProcessingTaskTypeId, + ['input' => $task->getInput()], + $task->getAppId(), + $task->getUserId(), + $task->getIdentifier(), + ); + + $task->setStatus(OCPTask::STATUS_RUNNING); + if ($task->getId() === null) { + $taskEntity = $this->taskMapper->insert(DbTask::fromPublicTask($task)); + $task->setId($taskEntity->getId()); + } else { + $this->taskMapper->update(DbTask::fromPublicTask($task)); + } + $this->logger->debug('Running a TextProcessing (' . $taskTypeClass . ') task with TaskProcessing'); + $taskProcessingResultTask = $this->taskProcessingManager->runTask($taskProcessingTask); + if ($taskProcessingResultTask->getStatus() === \OCP\TaskProcessing\Task::STATUS_SUCCESSFUL) { + $task->setOutput($taskProcessingResultTask->getOutput()['output'] ?? ''); + $task->setStatus(OCPTask::STATUS_SUCCESSFUL); + $this->taskMapper->update(DbTask::fromPublicTask($task)); + return $task->getOutput(); + } + } catch (\Throwable $e) { + $this->logger->error('TextProcessing to TaskProcessing failed', ['exception' => $e]); + $task->setStatus(OCPTask::STATUS_FAILED); + $this->taskMapper->update(DbTask::fromPublicTask($task)); + throw new TaskFailureException('TextProcessing to TaskProcessing failed: ' . $e->getMessage(), 0, $e); + } + $task->setStatus(OCPTask::STATUS_FAILED); + $this->taskMapper->update(DbTask::fromPublicTask($task)); + throw new TaskFailureException('Could not run task'); + } + + // try to run the text processing task if (!$this->canHandleTask($task)) { throw new PreConditionNotMetException('No text processing provider is installed that can handle this task'); } From aa64204075817b6b60132282b8b606955f7c5e83 Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Wed, 28 Aug 2024 17:26:32 +0200 Subject: [PATCH 02/12] feat(speech-to-text): SpeechToTextManager::transcribeFile calls TaskProcessingManager::runTask Signed-off-by: Julien Veyssier --- .../SpeechToText/SpeechToTextManager.php | 27 ++++++++++++++++++- lib/private/SpeechToText/TranscriptionJob.php | 2 +- lib/private/TaskProcessing/Manager.php | 26 ++++++++++++++++-- lib/private/TextProcessing/Manager.php | 11 +++++--- .../SpeechToText/ISpeechToTextManager.php | 4 ++- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/lib/private/SpeechToText/SpeechToTextManager.php b/lib/private/SpeechToText/SpeechToTextManager.php index d6cda47387567..c4397112a1815 100644 --- a/lib/private/SpeechToText/SpeechToTextManager.php +++ b/lib/private/SpeechToText/SpeechToTextManager.php @@ -24,6 +24,9 @@ use OCP\SpeechToText\ISpeechToTextProvider; use OCP\SpeechToText\ISpeechToTextProviderWithId; use OCP\SpeechToText\ISpeechToTextProviderWithUserId; +use OCP\TaskProcessing\IManager as ITaskProcessingManager; +use OCP\TaskProcessing\Task; +use OCP\TaskProcessing\TaskTypes\AudioToText; use Psr\Container\ContainerExceptionInterface; use Psr\Container\NotFoundExceptionInterface; use Psr\Log\LoggerInterface; @@ -41,6 +44,7 @@ public function __construct( private IJobList $jobList, private IConfig $config, private IUserSession $userSession, + private ITaskProcessingManager $taskProcessingManager, ) { } @@ -112,7 +116,28 @@ public function cancelScheduledFileTranscription(File $file, ?string $userId, st } } - public function transcribeFile(File $file): string { + public function transcribeFile(File $file, ?string $userId = null, string $appId = 'core'): string { + // try to run a TaskProcessing core:audio2text task + // this covers scheduling as well because OC\SpeechToText\TranscriptionJob calls this method + try { + $taskProcessingTask = new Task( + AudioToText::ID, + ['input' => $file->getId()], + $appId, + $userId, + 'from-SpeechToTextManager||' . $file->getId() . '||' . ($userId ?? '') . '||' . $appId, + ); + $resultTask = $this->taskProcessingManager->runTask($taskProcessingTask); + if ($resultTask->getStatus() === Task::STATUS_SUCCESSFUL) { + $output = $resultTask->getOutput(); + if (isset($output['output']) && is_string($output['output'])) { + return $output['output']; + } + } + } catch (Throwable $e) { + $this->logger->debug('Failed to run a Speech-to-text job from STTManager with TaskProcessing for file ' . $file->getId(), ['exception' => $e]); + } + if (!$this->hasProviders()) { throw new PreConditionNotMetException('No SpeechToText providers have been registered'); } diff --git a/lib/private/SpeechToText/TranscriptionJob.php b/lib/private/SpeechToText/TranscriptionJob.php index a46fd73786562..6e899ef6e9644 100644 --- a/lib/private/SpeechToText/TranscriptionJob.php +++ b/lib/private/SpeechToText/TranscriptionJob.php @@ -63,7 +63,7 @@ protected function run($argument) { ); return; } - $result = $this->speechToTextManager->transcribeFile($file); + $result = $this->speechToTextManager->transcribeFile($file, $userId, $appId); $this->eventDispatcher->dispatchTyped( new TranscriptionSuccessfulEvent( $fileId, diff --git a/lib/private/TaskProcessing/Manager.php b/lib/private/TaskProcessing/Manager.php index 3713d5915754d..5316414e8fb9f 100644 --- a/lib/private/TaskProcessing/Manager.php +++ b/lib/private/TaskProcessing/Manager.php @@ -88,7 +88,6 @@ public function __construct( IAppDataFactory $appDataFactory, private IRootFolder $rootFolder, private \OCP\TextToImage\IManager $textToImageManager, - private \OCP\SpeechToText\ISpeechToTextManager $speechToTextManager, private IUserMountCache $userMountCache, private IClientService $clientService, private IAppManager $appManager, @@ -369,12 +368,35 @@ public function getOptionalOutputShapeEnumValues(): array { return $newProviders; } + /** + * This is almost a copy of SpeechToTextManager->getProviders + * to avoid a dependency cycle between SpeechToTextManager and TaskProcessingManager + */ + private function _getRawSpeechToTextProviders(): array { + $context = $this->coordinator->getRegistrationContext(); + if ($context === null) { + return []; + } + $providers = []; + foreach ($context->getSpeechToTextProviders() as $providerServiceRegistration) { + $class = $providerServiceRegistration->getService(); + try { + $providers[$class] = $this->serverContainer->get($class); + } catch (NotFoundExceptionInterface|ContainerExceptionInterface|\Throwable $e) { + $this->logger->error('Failed to load SpeechToText provider ' . $class, [ + 'exception' => $e, + ]); + } + } + + return $providers; + } /** * @return IProvider[] */ private function _getSpeechToTextProviders(): array { - $oldProviders = $this->speechToTextManager->getProviders(); + $oldProviders = $this->_getRawSpeechToTextProviders(); $newProviders = []; foreach ($oldProviders as $oldProvider) { $newProvider = new class($oldProvider, $this->rootFolder, $this->appData) implements IProvider, ISynchronousProvider { diff --git a/lib/private/TextProcessing/Manager.php b/lib/private/TextProcessing/Manager.php index b9e89e1211b35..312efb29005ac 100644 --- a/lib/private/TextProcessing/Manager.php +++ b/lib/private/TextProcessing/Manager.php @@ -137,10 +137,13 @@ public function runTask(OCPTask $task): string { $this->logger->debug('Running a TextProcessing (' . $taskTypeClass . ') task with TaskProcessing'); $taskProcessingResultTask = $this->taskProcessingManager->runTask($taskProcessingTask); if ($taskProcessingResultTask->getStatus() === \OCP\TaskProcessing\Task::STATUS_SUCCESSFUL) { - $task->setOutput($taskProcessingResultTask->getOutput()['output'] ?? ''); - $task->setStatus(OCPTask::STATUS_SUCCESSFUL); - $this->taskMapper->update(DbTask::fromPublicTask($task)); - return $task->getOutput(); + $output = $taskProcessingResultTask->getOutput(); + if (isset($output['output']) && is_string($output['output'])) { + $task->setOutput($output['output']); + $task->setStatus(OCPTask::STATUS_SUCCESSFUL); + $this->taskMapper->update(DbTask::fromPublicTask($task)); + return $output['output']; + } } } catch (\Throwable $e) { $this->logger->error('TextProcessing to TaskProcessing failed', ['exception' => $e]); diff --git a/lib/public/SpeechToText/ISpeechToTextManager.php b/lib/public/SpeechToText/ISpeechToTextManager.php index 043dac0ba1443..6bd95197695b3 100644 --- a/lib/public/SpeechToText/ISpeechToTextManager.php +++ b/lib/public/SpeechToText/ISpeechToTextManager.php @@ -59,11 +59,13 @@ public function cancelScheduledFileTranscription(File $file, ?string $userId, st /** * @param File $file The media file to transcribe + * @param ?string $userId The user that triggered this request + * @param string $appId The app that triggered this request * @returns string The transcription of the passed media file * @throws PreConditionNotMetException If no provider was registered but this method was still called * @throws InvalidArgumentException If the file could not be found or is not of a supported type * @throws RuntimeException If the transcription failed for other reasons * @since 27.0.0 */ - public function transcribeFile(File $file): string; + public function transcribeFile(File $file, ?string $userId, string $appId): string; } From d2ec025fa492937ba7cf8c08b88baae2ef60c583 Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Thu, 29 Aug 2024 12:23:36 +0200 Subject: [PATCH 03/12] fix(taskprocessing): fix tests Signed-off-by: Julien Veyssier --- tests/lib/TaskProcessing/TaskProcessingTest.php | 12 ------------ tests/lib/TextProcessing/TextProcessingTest.php | 6 +++++- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/lib/TaskProcessing/TaskProcessingTest.php b/tests/lib/TaskProcessing/TaskProcessingTest.php index ac9dec1cd1dee..3b671c80feffe 100644 --- a/tests/lib/TaskProcessing/TaskProcessingTest.php +++ b/tests/lib/TaskProcessing/TaskProcessingTest.php @@ -26,7 +26,6 @@ use OCP\IServerContainer; use OCP\IUser; use OCP\IUserManager; -use OCP\SpeechToText\ISpeechToTextManager; use OCP\TaskProcessing\EShapeType; use OCP\TaskProcessing\Events\TaskFailedEvent; use OCP\TaskProcessing\Events\TaskSuccessfulEvent; @@ -450,15 +449,6 @@ protected function setUp(): void { $this->eventDispatcher = $this->createMock(IEventDispatcher::class); - $textProcessingManager = new \OC\TextProcessing\Manager( - $this->serverContainer, - $this->coordinator, - \OC::$server->get(LoggerInterface::class), - $this->jobList, - \OC::$server->get(\OC\TextProcessing\Db\TaskMapper::class), - \OC::$server->get(IConfig::class), - ); - $text2imageManager = new \OC\TextToImage\Manager( $this->serverContainer, $this->coordinator, @@ -481,9 +471,7 @@ protected function setUp(): void { $this->eventDispatcher, \OC::$server->get(IAppDataFactory::class), \OC::$server->get(IRootFolder::class), - $textProcessingManager, $text2imageManager, - \OC::$server->get(ISpeechToTextManager::class), $this->userMountCache, \OC::$server->get(IClientService::class), \OC::$server->get(IAppManager::class), diff --git a/tests/lib/TextProcessing/TextProcessingTest.php b/tests/lib/TextProcessing/TextProcessingTest.php index db479a7a7b63d..9c759e524580e 100644 --- a/tests/lib/TextProcessing/TextProcessingTest.php +++ b/tests/lib/TextProcessing/TextProcessingTest.php @@ -86,6 +86,9 @@ public function getTaskType(): string { } } +/** + * @group DB + */ class TextProcessingTest extends \Test\TestCase { private IManager $manager; private Coordinator $coordinator; @@ -176,7 +179,8 @@ protected function setUp(): void { \OC::$server->get(LoggerInterface::class), $this->jobList, $this->taskMapper, - $config + $config, + \OC::$server->get(\OCP\TaskProcessing\IManager::class), ); } From f54eb30232ff75db860c57c3f9b38fb09d70d980 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 30 Aug 2024 08:26:55 +0200 Subject: [PATCH 04/12] fix(TaskProcessing): Use OCP\Server::get instead of copying methods Signed-off-by: Marcel Klehr Signed-off-by: Julien Veyssier --- lib/private/TaskProcessing/Manager.php | 50 +++----------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/lib/private/TaskProcessing/Manager.php b/lib/private/TaskProcessing/Manager.php index 5316414e8fb9f..634a50ac7e675 100644 --- a/lib/private/TaskProcessing/Manager.php +++ b/lib/private/TaskProcessing/Manager.php @@ -36,6 +36,7 @@ use OCP\IServerContainer; use OCP\L10N\IFactory; use OCP\Lock\LockedException; +use OCP\SpeechToText\ISpeechToTextManager; use OCP\SpeechToText\ISpeechToTextProvider; use OCP\SpeechToText\ISpeechToTextProviderWithId; use OCP\TaskProcessing\EShapeType; @@ -95,31 +96,9 @@ public function __construct( $this->appData = $appDataFactory->get('core'); } - - /** - * This is almost a copy of textProcessingManager->getProviders - * to avoid a dependency cycle between TextProcessingManager and TaskProcessingManager - */ private function _getRawTextProcessingProviders(): array { - $context = $this->coordinator->getRegistrationContext(); - if ($context === null) { - return []; - } - - $providers = []; - - foreach ($context->getTextProcessingProviders() as $providerServiceRegistration) { - $class = $providerServiceRegistration->getService(); - try { - $providers[$class] = $this->serverContainer->get($class); - } catch (\Throwable $e) { - $this->logger->error('Failed to load Text processing provider ' . $class, [ - 'exception' => $e, - ]); - } - } - - return $providers; + $textProcessingManager = \OCP\Server::get(\OCP\TextProcessing\IManager::class); + return $textProcessingManager->getProviders(); } private function _getTextProcessingProviders(): array { @@ -368,28 +347,9 @@ public function getOptionalOutputShapeEnumValues(): array { return $newProviders; } - /** - * This is almost a copy of SpeechToTextManager->getProviders - * to avoid a dependency cycle between SpeechToTextManager and TaskProcessingManager - */ private function _getRawSpeechToTextProviders(): array { - $context = $this->coordinator->getRegistrationContext(); - if ($context === null) { - return []; - } - $providers = []; - foreach ($context->getSpeechToTextProviders() as $providerServiceRegistration) { - $class = $providerServiceRegistration->getService(); - try { - $providers[$class] = $this->serverContainer->get($class); - } catch (NotFoundExceptionInterface|ContainerExceptionInterface|\Throwable $e) { - $this->logger->error('Failed to load SpeechToText provider ' . $class, [ - 'exception' => $e, - ]); - } - } - - return $providers; + $speechToTextManager = \OCP\Server::get(ISpeechToTextManager::class); + return $speechToTextManager->getProviders(); } /** From 578e576b89d7d60adbe4d07d706bb4da006d0c5e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 30 Aug 2024 08:33:06 +0200 Subject: [PATCH 05/12] fix(SpeechToTextManager): Throw TaskProcessing Task failed Signed-off-by: Marcel Klehr Signed-off-by: Julien Veyssier --- .../SpeechToText/SpeechToTextManager.php | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/private/SpeechToText/SpeechToTextManager.php b/lib/private/SpeechToText/SpeechToTextManager.php index c4397112a1815..d69add2d80bea 100644 --- a/lib/private/SpeechToText/SpeechToTextManager.php +++ b/lib/private/SpeechToText/SpeechToTextManager.php @@ -120,22 +120,24 @@ public function transcribeFile(File $file, ?string $userId = null, string $appId // try to run a TaskProcessing core:audio2text task // this covers scheduling as well because OC\SpeechToText\TranscriptionJob calls this method try { - $taskProcessingTask = new Task( - AudioToText::ID, - ['input' => $file->getId()], - $appId, - $userId, - 'from-SpeechToTextManager||' . $file->getId() . '||' . ($userId ?? '') . '||' . $appId, - ); - $resultTask = $this->taskProcessingManager->runTask($taskProcessingTask); - if ($resultTask->getStatus() === Task::STATUS_SUCCESSFUL) { - $output = $resultTask->getOutput(); - if (isset($output['output']) && is_string($output['output'])) { - return $output['output']; + if (isset($this->taskProcessingManager->getAvailableTaskTypes()['core:audio2text'])) { + $taskProcessingTask = new Task( + AudioToText::ID, + ['input' => $file->getId()], + $appId, + $userId, + 'from-SpeechToTextManager||' . $file->getId() . '||' . ($userId ?? '') . '||' . $appId, + ); + $resultTask = $this->taskProcessingManager->runTask($taskProcessingTask); + if ($resultTask->getStatus() === Task::STATUS_SUCCESSFUL) { + $output = $resultTask->getOutput(); + if (isset($output['output']) && is_string($output['output'])) { + return $output['output']; + } } } } catch (Throwable $e) { - $this->logger->debug('Failed to run a Speech-to-text job from STTManager with TaskProcessing for file ' . $file->getId(), ['exception' => $e]); + throw new RuntimeException('Failed to run a Speech-to-text job from STTManager with TaskProcessing for file ' . $file->getId(), 0, $e); } if (!$this->hasProviders()) { From 64468b847dc8aa0bec00bb96477be2401f51f888 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 30 Aug 2024 09:15:59 +0200 Subject: [PATCH 06/12] fix(settings): Remove STT admin settings taskprocessing is transparent to STT providers so specific STT settings are obsolete Signed-off-by: Marcel Klehr Signed-off-by: Julien Veyssier --- apps/settings/src/components/AdminAI.vue | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/apps/settings/src/components/AdminAI.vue b/apps/settings/src/components/AdminAI.vue index dc64f64e65b95..7a7ddca04c5de 100644 --- a/apps/settings/src/components/AdminAI.vue +++ b/apps/settings/src/components/AdminAI.vue @@ -50,24 +50,6 @@ - - - -