From c8cab9d2fd347975d3d5b4e1d7d9b549e5db6e25 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 7 Sep 2023 12:37:09 +0200 Subject: [PATCH 01/66] Implement TextToImage OCP API Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 221 ++++++++++++++++ .../Version28000Date20230906104802.php | 100 +++++++ core/ResponseDefinitions.php | 9 + .../Bootstrap/RegistrationContext.php | 24 ++ .../Repair/AddRemoveOldTasksBackgroundJob.php | 8 +- lib/private/TextToImage/Db/Task.php | 119 +++++++++ lib/private/TextToImage/Db/TaskMapper.php | 118 +++++++++ lib/private/TextToImage/Manager.php | 245 ++++++++++++++++++ .../RemoveOldTasksBackgroundJob.php | 59 +++++ lib/private/TextToImage/TaskBackgroundJob.php | 63 +++++ .../Events/AbstractTextToImageEvent.php | 52 ++++ .../TextToImage/Events/TaskFailedEvent.php | 54 ++++ .../Events/TaskSuccessfulEvent.php | 33 +++ .../TextToImage/Exception/Exception.php | 29 +++ .../Exception/TaskNotFoundException.php | 29 +++ lib/public/TextToImage/IManager.php | 98 +++++++ lib/public/TextToImage/IProvider.php | 52 ++++ lib/public/TextToImage/Task.php | 179 +++++++++++++ 18 files changed, 1489 insertions(+), 3 deletions(-) create mode 100644 core/Controller/TextToImageApiController.php create mode 100644 core/Migrations/Version28000Date20230906104802.php create mode 100644 lib/private/TextToImage/Db/Task.php create mode 100644 lib/private/TextToImage/Db/TaskMapper.php create mode 100644 lib/private/TextToImage/Manager.php create mode 100644 lib/private/TextToImage/RemoveOldTasksBackgroundJob.php create mode 100644 lib/private/TextToImage/TaskBackgroundJob.php create mode 100644 lib/public/TextToImage/Events/AbstractTextToImageEvent.php create mode 100644 lib/public/TextToImage/Events/TaskFailedEvent.php create mode 100644 lib/public/TextToImage/Events/TaskSuccessfulEvent.php create mode 100644 lib/public/TextToImage/Exception/Exception.php create mode 100644 lib/public/TextToImage/Exception/TaskNotFoundException.php create mode 100644 lib/public/TextToImage/IManager.php create mode 100644 lib/public/TextToImage/IProvider.php create mode 100644 lib/public/TextToImage/Task.php diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php new file mode 100644 index 0000000000000..4d2f6c81c90b2 --- /dev/null +++ b/core/Controller/TextToImageApiController.php @@ -0,0 +1,221 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +namespace OC\Core\Controller; + +use OC\Files\AppData\AppData; +use OCA\Core\ResponseDefinitions; +use OCP\AppFramework\Http; +use OCP\AppFramework\Http\Attribute\AnonRateLimit; +use OCP\AppFramework\Http\Attribute\NoAdminRequired; +use OCP\AppFramework\Http\Attribute\PublicPage; +use OCP\AppFramework\Http\Attribute\UserRateLimit; +use OCP\AppFramework\Http\DataResponse; +use OCP\AppFramework\Http\FileDisplayResponse; +use OCP\IL10N; +use OCP\IRequest; +use OCP\TextToImage\Exception\TaskNotFoundException; +use OCP\TextToImage\Task; +use OCP\TextToImage\IManager; +use OCP\PreConditionNotMetException; + +/** + * @psalm-import-type CoreTextToImageTask from ResponseDefinitions + */ +class TextToImageApiController extends \OCP\AppFramework\OCSController { + public function __construct( + string $appName, + IRequest $request, + private IManager $textToImageManager, + private IL10N $l, + private ?string $userId, + private AppData $appData, + ) { + parent::__construct($appName, $request); + } + + /** + * @PublicPage + * + * Check whether this feature is available + * + * @return DataResponse + */ + public function isAvailable(): DataResponse { + return new DataResponse([ + 'isAvailable' => $this->textToImageManager->hasProviders(), + ]); + } + + /** + * This endpoint allows scheduling a text to image task + * + * @param string $input Input text + * @param string $appId ID of the app that will execute the task + * @param string $identifier An arbitrary identifier for the task + * + * @return DataResponse|DataResponse + * + * 200: Task scheduled successfully + * 400: Scheduling task is not possible + * 412: Scheduling task is not possible + */ + #[PublicPage] + #[UserRateLimit(limit: 20, period: 120)] + #[AnonRateLimit(limit: 5, period: 120)] + public function schedule(string $input, string $type, string $appId, string $identifier = ''): DataResponse { + $task = new Task($input, $appId, $this->userId, $identifier); + try { + $this->textToImageManager->scheduleTask($task); + + $json = $task->jsonSerialize(); + + return new DataResponse([ + 'task' => $json, + ]); + } catch (PreConditionNotMetException) { + return new DataResponse(['message' => $this->l->t('No text to image provider is available')], Http::STATUS_PRECONDITION_FAILED); + } + } + + /** + * This endpoint allows checking the status and results of a task. + * Tasks are removed 1 week after receiving their last update. + * + * @param int $id The id of the task + * + * @return DataResponse|DataResponse + * + * 200: Task returned + * 404: Task not found + */ + #[PublicPage] + public function getTask(int $id): DataResponse { + try { + $task = $this->textToImageManager->getUserTask($id, $this->userId); + + $json = $task->jsonSerialize(); + + return new DataResponse([ + 'task' => $json, + ]); + } catch (TaskNotFoundException $e) { + return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + } catch (\RuntimeException $e) { + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + } + + /** + * This endpoint allows checking the status and results of a task. + * Tasks are removed 1 week after receiving their last update. + * + * @param int $id The id of the task + * + * @return FileDisplayResponse|DataResponse + * + * 200: Task returned + * 404: Task not found + */ + #[PublicPage] + public function getImage(int $id): DataResponse|FileDisplayResponse { + try { + $task = $this->textToImageManager->getUserTask($id, $this->userId); + try { + $folder = $this->appData->getFolder('text2image'); + } catch(\OCP\Files\NotFoundException) { + $folder = $this->appData->newFolder('text2image'); + } + $file = $folder->getFile((string)$task->getId()); + $info = getimagesizefromstring($file->getContent()); + + return new FileDisplayResponse($file, Http::STATUS_OK, ['Content-Type' => image_type_to_mime_type($info[2])]); + } catch (TaskNotFoundException) { + return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + } catch (\RuntimeException) { + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + } catch (\OCP\Files\NotFoundException) { + return new DataResponse(['message' => $this->l->t('Image not found')], Http::STATUS_NOT_FOUND); + } + } + + /** + * This endpoint allows to delete a scheduled task for a user + * + * @param int $id The id of the task + * + * @return DataResponse|DataResponse + * + * 200: Task returned + * 404: Task not found + */ + #[NoAdminRequired] + public function deleteTask(int $id): DataResponse { + try { + $task = $this->textToImageManager->getUserTask($id, $this->userId); + + $this->textToImageManager->deleteTask($task); + + $json = $task->jsonSerialize(); + + return new DataResponse([ + 'task' => $json, + ]); + } catch (TaskNotFoundException $e) { + return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + } catch (\RuntimeException $e) { + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + } + + + /** + * This endpoint returns a list of tasks of a user that are related + * with a specific appId and optionally with an identifier + * + * @param string $appId ID of the app + * @param string|null $identifier An arbitrary identifier for the task + * @return DataResponse|DataResponse + * + * 200: Task list returned + */ + #[NoAdminRequired] + public function listTasksByApp(string $appId, ?string $identifier = null): DataResponse { + try { + $tasks = $this->textToImageManager->getUserTasksByApp($this->userId, $appId, $identifier); + /** @var CoreTextToImageTask[] $json */ + $json = array_map(static function (Task $task) { + return $task->jsonSerialize(); + }, $tasks); + + return new DataResponse([ + 'tasks' => $json, + ]); + } catch (\RuntimeException $e) { + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + } +} diff --git a/core/Migrations/Version28000Date20230906104802.php b/core/Migrations/Version28000Date20230906104802.php new file mode 100644 index 0000000000000..662bdd648b7d3 --- /dev/null +++ b/core/Migrations/Version28000Date20230906104802.php @@ -0,0 +1,100 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +/** + * Introduce text2image_tasks table + */ +class Version28000Date20230906104802 extends SimpleMigrationStep { + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + $changed = false; + if (!$schema->hasTable('text2image_tasks')) { + $table = $schema->createTable('text2image_tasks'); + + $table->addColumn('id', Types::BIGINT, [ + 'notnull' => true, + 'length' => 64, + 'autoincrement' => true, + ]); + $table->addColumn('input', Types::TEXT, [ + 'notnull' => true, + ]); + $table->addColumn('status', Types::INTEGER, [ + 'notnull' => false, + 'length' => 6, + 'default' => 0, + ]); + $table->addColumn('user_id', Types::STRING, [ + 'notnull' => false, + 'length' => 64, + ]); + $table->addColumn('app_id', Types::STRING, [ + 'notnull' => true, + 'length' => 32, + 'default' => '', + ]); + $table->addColumn('identifier', Types::STRING, [ + 'notnull' => true, + 'length' => 255, + 'default' => '', + ]); + $table->addColumn('last_updated', Types::INTEGER, [ + 'notnull' => false, + 'length' => 4, + 'default' => 0, + 'unsigned' => true, + ]); + + $table->setPrimaryKey(['id'], 't2i_tasks_id_index'); + $table->addIndex(['last_updated'], 't2i_tasks_updated'); + $table->addIndex(['status'], 't2i_tasks_status'); + $table->addIndex(['user_id', 'app_id', 'identifier'], 't2i_tasks_uid_appid_ident'); + + $changed = true; + } + + if ($changed) { + return $schema; + } + + return null; + } +} diff --git a/core/ResponseDefinitions.php b/core/ResponseDefinitions.php index 7e2bc643ce564..97c91c5bbe0d2 100644 --- a/core/ResponseDefinitions.php +++ b/core/ResponseDefinitions.php @@ -144,6 +144,15 @@ * output: ?string, * identifier: string, * } + * + * @psalm-type CoreTextToImageTask = array{ + * id: ?int, + * status: 0|1|2|3|4, + * userId: ?string, + * appId: string, + * input: string, + * identifier: string, + * } */ class ResponseDefinitions { } diff --git a/lib/private/AppFramework/Bootstrap/RegistrationContext.php b/lib/private/AppFramework/Bootstrap/RegistrationContext.php index b3ef3ee65fba6..462b7bb237fba 100644 --- a/lib/private/AppFramework/Bootstrap/RegistrationContext.php +++ b/lib/private/AppFramework/Bootstrap/RegistrationContext.php @@ -137,6 +137,12 @@ class RegistrationContext { /** @var ServiceRegistration[] */ private array $referenceProviders = []; + /** @var ServiceRegistration<\OCP\TextToImage\IProvider>[] */ + private $textToImageProviders = []; + + + + /** @var ParameterRegistration[] */ private $sensitiveMethods = []; @@ -270,6 +276,13 @@ public function registerTextProcessingProvider(string $providerClass): void { ); } + public function registerTextToImageProvider(string $providerClass): void { + $this->context->registerTextToImageProvider( + $this->appId, + $providerClass + ); + } + public function registerTemplateProvider(string $providerClass): void { $this->context->registerTemplateProvider( $this->appId, @@ -440,6 +453,10 @@ public function registerTextProcessingProvider(string $appId, string $class): vo $this->textProcessingProviders[] = new ServiceRegistration($appId, $class); } + public function registerTextToImageProvider(string $appId, string $class): void { + $this->textToImageProviders[] = new ServiceRegistration($appId, $class); + } + public function registerTemplateProvider(string $appId, string $class): void { $this->templateProviders[] = new ServiceRegistration($appId, $class); } @@ -722,6 +739,13 @@ public function getTextProcessingProviders(): array { return $this->textProcessingProviders; } + /** + * @return ServiceRegistration<\OCP\TextToImage\IProvider>[] + */ + public function getTextToImageProviders(): array { + return $this->textToImageProviders; + } + /** * @return ServiceRegistration[] */ diff --git a/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php b/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php index 94ae39f2183e2..00badbb726dd6 100644 --- a/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php +++ b/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php @@ -25,7 +25,8 @@ */ namespace OC\Repair; -use OC\TextProcessing\RemoveOldTasksBackgroundJob; +use OC\TextProcessing\RemoveOldTasksBackgroundJob as RemoveOldTextProcessingTasksBackgroundJob; +use OC\TextToImage\RemoveOldTasksBackgroundJob as RemoveOldTextToImageTasksBackgroundJob; use OCP\BackgroundJob\IJobList; use OCP\Migration\IOutput; use OCP\Migration\IRepairStep; @@ -38,10 +39,11 @@ public function __construct(IJobList $jobList) { } public function getName(): string { - return 'Add language model tasks cleanup job'; + return 'Add AI tasks cleanup job'; } public function run(IOutput $output) { - $this->jobList->add(RemoveOldTasksBackgroundJob::class); + $this->jobList->add(RemoveOldTextProcessingTasksBackgroundJob::class); + $this->jobList->add(RemoveOldTextToImageTasksBackgroundJob::class); } } diff --git a/lib/private/TextToImage/Db/Task.php b/lib/private/TextToImage/Db/Task.php new file mode 100644 index 0000000000000..84d43ab6fd999 --- /dev/null +++ b/lib/private/TextToImage/Db/Task.php @@ -0,0 +1,119 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OC\TextToImage\Db; + +use OCP\AppFramework\Db\Entity; +use OCP\Files\AppData\IAppDataFactory; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; +use OCP\Image; +use OCP\TextToImage\Task as OCPTask; + +/** + * @method setLastUpdated(int $lastUpdated) + * @method int getLastUpdated() + * @method setInput(string $type) + * @method string getInput() + * @method setResultPath(string $resultPath) + * @method string getResultPath() + * @method setStatus(int $type) + * @method int getStatus() + * @method setUserId(?string $userId) + * @method string|null getUserId() + * @method setAppId(string $type) + * @method string getAppId() + * @method setIdentifier(string $identifier) + * @method string getIdentifier() + */ +class Task extends Entity { + protected $lastUpdated; + protected $type; + protected $input; + protected $status; + protected $userId; + protected $appId; + protected $identifier; + + /** + * @var string[] + */ + public static array $columns = ['id', 'last_updated', 'input', 'status', 'user_id', 'app_id', 'identifier']; + + /** + * @var string[] + */ + public static array $fields = ['id', 'lastUpdated', 'input', 'status', 'userId', 'appId', 'identifier']; + + + public function __construct() { + // add types in constructor + $this->addType('id', 'integer'); + $this->addType('lastUpdated', 'integer'); + $this->addType('input', 'string'); + $this->addType('status', 'integer'); + $this->addType('userId', 'string'); + $this->addType('appId', 'string'); + $this->addType('identifier', 'string'); + } + + public function toRow(): array { + return array_combine(self::$columns, array_map(function ($field) { + return $this->{'get'.ucfirst($field)}(); + }, self::$fields)); + } + + public static function fromPublicTask(OCPTask $task): Task { + /** @var Task $dbTask */ + $dbTask = Task::fromParams([ + 'id' => $task->getId(), + 'lastUpdated' => time(), + 'status' => $task->getStatus(), + 'input' => $task->getInput(), + 'userId' => $task->getUserId(), + 'appId' => $task->getAppId(), + 'identifier' => $task->getIdentifier(), + ]); + return $dbTask; + } + + public function toPublicTask(): OCPTask { + $task = new OCPTask($this->getInput(), $this->getAppId(), $this->getuserId(), $this->getIdentifier()); + $task->setId($this->getId()); + $task->setStatus($this->getStatus()); + $appData = \OC::$server->get(IAppDataFactory::class)->get('core'); + try { + try { + $folder = $appData->getFolder('text2image'); + } catch(NotFoundException) { + $folder = $appData->newFolder('text2image'); + } + $task->setOutputImage(new Image(base64_encode($folder->getFile((string)$task->getId())->getContent()))); + } catch (NotFoundException|NotPermittedException) { + // noop + } + return $task; + } +} diff --git a/lib/private/TextToImage/Db/TaskMapper.php b/lib/private/TextToImage/Db/TaskMapper.php new file mode 100644 index 0000000000000..44d8aea24eb4a --- /dev/null +++ b/lib/private/TextToImage/Db/TaskMapper.php @@ -0,0 +1,118 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OC\TextToImage\Db; + +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\Entity; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; +use OCP\AppFramework\Db\QBMapper; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\DB\Exception; +use OCP\IDBConnection; + +/** + * @extends QBMapper + */ +class TaskMapper extends QBMapper { + public function __construct( + IDBConnection $db, + private ITimeFactory $timeFactory, + ) { + parent::__construct($db, 'text2image_tasks', Task::class); + } + + /** + * @param int $id + * @return Task + * @throws Exception + * @throws DoesNotExistException + * @throws MultipleObjectsReturnedException + */ + public function find(int $id): Task { + $qb = $this->db->getQueryBuilder(); + $qb->select(Task::$columns) + ->from($this->tableName) + ->where($qb->expr()->eq('id', $qb->createPositionalParameter($id))); + return $this->findEntity($qb); + } + + /** + * @param int $id + * @param string|null $userId + * @return Task + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + */ + public function findByIdAndUser(int $id, ?string $userId): Task { + $qb = $this->db->getQueryBuilder(); + $qb->select(Task::$columns) + ->from($this->tableName) + ->where($qb->expr()->eq('id', $qb->createPositionalParameter($id))); + if ($userId === null) { + $qb->andWhere($qb->expr()->isNull('user_id')); + } else { + $qb->andWhere($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId))); + } + return $this->findEntity($qb); + } + + /** + * @param string $userId + * @param string $appId + * @param string|null $identifier + * @return array + * @throws Exception + */ + public function findUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array { + $qb = $this->db->getQueryBuilder(); + $qb->select(Task::$columns) + ->from($this->tableName) + ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId))) + ->andWhere($qb->expr()->eq('app_id', $qb->createPositionalParameter($appId))); + if ($identifier !== null) { + $qb->andWhere($qb->expr()->eq('identifier', $qb->createPositionalParameter($identifier))); + } + return $this->findEntities($qb); + } + + /** + * @param int $timeout + * @return int the number of deleted tasks + * @throws Exception + */ + public function deleteOlderThan(int $timeout): int { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->tableName) + ->where($qb->expr()->lt('last_updated', $qb->createPositionalParameter(time() - $timeout))); + return $qb->executeStatement(); + } + + public function update(Entity $entity): Entity { + $entity->setLastUpdated($this->timeFactory->now()->getTimestamp()); + return parent::update($entity); + } +} diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php new file mode 100644 index 0000000000000..8700ea8b5675d --- /dev/null +++ b/lib/private/TextToImage/Manager.php @@ -0,0 +1,245 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OC\TextToImage; + +use OC\AppFramework\Bootstrap\Coordinator; +use OC\TextToImage\Db\Task as DbTask; +use OCP\Files\AppData\IAppDataFactory; +use OCP\Files\IAppData; +use OCP\IConfig; +use OCP\TextToImage\Exception\TaskNotFoundException; +use OCP\TextToImage\IManager; +use OCP\TextToImage\Task; +use OC\TextToImage\Db\TaskMapper; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; +use OCP\BackgroundJob\IJobList; +use OCP\DB\Exception; +use OCP\IServerContainer; +use OCP\TextToImage\IProvider; +use OCP\PreConditionNotMetException; +use Psr\Log\LoggerInterface; +use RuntimeException; +use Throwable; + +class Manager implements IManager { + /** @var ?IProvider[] */ + private ?array $providers = null; + private IAppData $appData; + + public function __construct( + private IServerContainer $serverContainer, + private Coordinator $coordinator, + private LoggerInterface $logger, + private IJobList $jobList, + private TaskMapper $taskMapper, + private IConfig $config, + private IAppDataFactory $appDataFactory, + ) { + $this->appData = $this->appDataFactory->get('core'); + } + + public function getProviders(): array { + $context = $this->coordinator->getRegistrationContext(); + if ($context === null) { + return []; + } + + if ($this->providers !== null) { + return $this->providers; + } + + $this->providers = []; + + foreach ($context->getTextToImageProviders() as $providerServiceRegistration) { + $class = $providerServiceRegistration->getService(); + try { + $this->providers[$class] = $this->serverContainer->get($class); + } catch (Throwable $e) { + $this->logger->error('Failed to load Text to image provider ' . $class, [ + 'exception' => $e, + ]); + } + } + + return $this->providers; + } + + public function hasProviders(): bool { + $context = $this->coordinator->getRegistrationContext(); + if ($context === null) { + return false; + } + return count($context->getTextToImageProviders()) > 0; + } + + /** + * @inheritDoc + */ + public function runTask(Task $task): void { + if (!$this->hasProviders()) { + throw new PreConditionNotMetException('No text to image provider is installed that can handle this task'); + } + $providers = $this->getProviders(); + + $json = $this->config->getAppValue('core', 'ai.text2image_provider', ''); + if ($json !== '') { + $className = json_decode($json, true); + $provider = current(array_filter($providers, fn ($provider) => $provider::class === $className)); + if ($provider !== false) { + $providers = [$provider]; + } + } + + foreach ($providers as $provider) { + try { + $task->setStatus(Task::STATUS_RUNNING); + if ($task->getId() === null) { + $taskEntity = $this->taskMapper->insert(DbTask::fromPublicTask($task)); + $task->setId($taskEntity->getId()); + } else { + $this->taskMapper->update(DbTask::fromPublicTask($task)); + } + try { + $folder = $this->appData->getFolder('text2image'); + } catch(\OCP\Files\NotFoundException $e) { + $folder = $this->appData->newFolder('text2image'); + } + $file = $folder->newFile((string) $task->getId()); + $provider->generate($task->getInput(), $file->write()); + $task->setResultPath($file->getName()); + $task->setStatus(Task::STATUS_SUCCESSFUL); + $this->taskMapper->update(DbTask::fromPublicTask($task)); + return; + } catch (\RuntimeException $e) { + $this->logger->info('Text2Image generation using provider ' . $provider->getName() . ' failed', ['exception' => $e]); + $task->setStatus(Task::STATUS_FAILED); + $this->taskMapper->update(DbTask::fromPublicTask($task)); + throw $e; + } catch (\Throwable $e) { + $this->logger->info('Text2Image generation using provider ' . $provider->getName() . ' failed', ['exception' => $e]); + $task->setStatus(Task::STATUS_FAILED); + $this->taskMapper->update(DbTask::fromPublicTask($task)); + throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: ' . $e->getMessage(), 0, $e); + } + } + + throw new RuntimeException('Could not run task'); + } + + /** + * @inheritDoc + * @throws Exception + */ + public function scheduleTask(Task $task): void { + if (!$this->hasProviders()) { + throw new PreConditionNotMetException('No text to image provider is installed that can handle this task'); + } + $task->setStatus(Task::STATUS_SCHEDULED); + $taskEntity = DbTask::fromPublicTask($task); + $this->taskMapper->insert($taskEntity); + $task->setId($taskEntity->getId()); + $this->jobList->add(TaskBackgroundJob::class, [ + 'taskId' => $task->getId() + ]); + } + + /** + * @inheritDoc + */ + public function deleteTask(Task $task): void { + $taskEntity = DbTask::fromPublicTask($task); + $this->taskMapper->delete($taskEntity); + $this->jobList->remove(TaskBackgroundJob::class, [ + 'taskId' => $task->getId() + ]); + } + + /** + * Get a task from its id + * + * @param int $id The id of the task + * @return Task + * @throws RuntimeException If the query failed + * @throws NotFoundException If the task could not be found + */ + public function getTask(int $id): Task { + try { + $taskEntity = $this->taskMapper->find($id); + return $taskEntity->toPublicTask(); + } catch (DoesNotExistException $e) { + throw new TaskNotFoundException('Could not find task with the provided id'); + } catch (MultipleObjectsReturnedException $e) { + throw new RuntimeException('Could not uniquely identify task with given id', 0, $e); + } catch (Exception $e) { + throw new RuntimeException('Failure while trying to find task by id: ' . $e->getMessage(), 0, $e); + } + } + + /** + * Get a task from its user id and task id + * If userId is null, this can only get a task that was scheduled anonymously + * + * @param int $id The id of the task + * @param string|null $userId The user id that scheduled the task + * @return Task + * @throws RuntimeException If the query failed + * @throws NotFoundException If the task could not be found + */ + public function getUserTask(int $id, ?string $userId): Task { + try { + $taskEntity = $this->taskMapper->findByIdAndUser($id, $userId); + return $taskEntity->toPublicTask(); + } catch (DoesNotExistException $e) { + throw new TaskNotFoundException('Could not find task with the provided id and user id'); + } catch (MultipleObjectsReturnedException $e) { + throw new RuntimeException('Could not uniquely identify task with given id and user id', 0, $e); + } catch (Exception $e) { + throw new RuntimeException('Failure while trying to find task by id and user id: ' . $e->getMessage(), 0, $e); + } + } + + /** + * Get a list of tasks scheduled by a specific user for a specific app + * and optionally with a specific identifier. + * This cannot be used to get anonymously scheduled tasks + * + * @param string $userId + * @param string $appId + * @param string|null $identifier + * @return array + */ + public function getUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array { + try { + $taskEntities = $this->taskMapper->findUserTasksByApp($userId, $appId, $identifier); + return array_map(static function (DbTask $taskEntity) { + return $taskEntity->toPublicTask(); + }, $taskEntities); + } catch (Exception $e) { + throw new RuntimeException('Failure while trying to find tasks by appId and identifier: ' . $e->getMessage(), 0, $e); + } + } +} diff --git a/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php b/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php new file mode 100644 index 0000000000000..fe6c77cb7902e --- /dev/null +++ b/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php @@ -0,0 +1,59 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +namespace OC\TextToImage; + +use OC\TextToImage\Db\TaskMapper; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\TimedJob; +use OCP\DB\Exception; +use Psr\Log\LoggerInterface; + +class RemoveOldTasksBackgroundJob extends TimedJob { + public const MAX_TASK_AGE_SECONDS = 60 * 50 * 24 * 7; // 1 week + + public function __construct( + ITimeFactory $timeFactory, + private TaskMapper $taskMapper, + private LoggerInterface $logger, + + ) { + parent::__construct($timeFactory); + $this->setInterval(60 * 60 * 24); + } + + /** + * @param mixed $argument + * @inheritDoc + */ + protected function run($argument) { + try { + $this->taskMapper->deleteOlderThan(self::MAX_TASK_AGE_SECONDS); + } catch (Exception $e) { + $this->logger->warning('Failed to delete stale text to image tasks', ['exception' => $e]); + } + } +} diff --git a/lib/private/TextToImage/TaskBackgroundJob.php b/lib/private/TextToImage/TaskBackgroundJob.php new file mode 100644 index 0000000000000..b223258485a84 --- /dev/null +++ b/lib/private/TextToImage/TaskBackgroundJob.php @@ -0,0 +1,63 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +namespace OC\TextToImage; + +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\TextToImage\Events\TaskFailedEvent; +use OCP\TextToImage\Events\TaskSuccessfulEvent; +use OCP\TextToImage\IManager; + +class TaskBackgroundJob extends QueuedJob { + public function __construct( + ITimeFactory $timeFactory, + private IManager $text2imageManager, + private IEventDispatcher $eventDispatcher, + ) { + parent::__construct($timeFactory); + // We want to avoid overloading the machine with these jobs + // so we only allow running one job at a time + $this->setAllowParallelRuns(false); + } + + /** + * @param array{taskId: int} $argument + * @inheritDoc + */ + protected function run($argument) { + $taskId = $argument['taskId']; + $task = $this->text2imageManager->getTask($taskId); + try { + $this->text2imageManager->runTask($task); + $event = new TaskSuccessfulEvent($task); + } catch (\Throwable $e) { + $event = new TaskFailedEvent($task, $e->getMessage()); + } + $this->eventDispatcher->dispatchTyped($event); + } +} diff --git a/lib/public/TextToImage/Events/AbstractTextToImageEvent.php b/lib/public/TextToImage/Events/AbstractTextToImageEvent.php new file mode 100644 index 0000000000000..56c68195602b3 --- /dev/null +++ b/lib/public/TextToImage/Events/AbstractTextToImageEvent.php @@ -0,0 +1,52 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\TextToImage\Events; + +use OCP\EventDispatcher\Event; +use OCP\TextToImage\Task; + +/** + * @since 28.0.0 + */ +abstract class AbstractTextToImageEvent extends Event { + /** + * @since 28.0.0 + */ + public function __construct( + private Task $task + ) { + parent::__construct(); + } + + /** + * @return Task + * @since 28.0.0 + */ + public function getTask(): Task { + return $this->task; + } +} diff --git a/lib/public/TextToImage/Events/TaskFailedEvent.php b/lib/public/TextToImage/Events/TaskFailedEvent.php new file mode 100644 index 0000000000000..0d91b3a4f6791 --- /dev/null +++ b/lib/public/TextToImage/Events/TaskFailedEvent.php @@ -0,0 +1,54 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\TextToImage\Events; + +use OCP\TextToImage\Task; + +/** + * @since 28.0.0 + */ +class TaskFailedEvent extends AbstractTextToImageEvent { + /** + * @param Task $task + * @param string $errorMessage + * @since 28.0.0 + */ + public function __construct( + Task $task, + private string $errorMessage, + ) { + parent::__construct($task); + } + + /** + * @return string + * @since 28.0.0 + */ + public function getErrorMessage(): string { + return $this->errorMessage; + } +} diff --git a/lib/public/TextToImage/Events/TaskSuccessfulEvent.php b/lib/public/TextToImage/Events/TaskSuccessfulEvent.php new file mode 100644 index 0000000000000..3e2e76198daa5 --- /dev/null +++ b/lib/public/TextToImage/Events/TaskSuccessfulEvent.php @@ -0,0 +1,33 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\TextToImage\Events; + +/** + * @since 27.1.0 + */ +class TaskSuccessfulEvent extends AbstractTextToImageEvent { +} diff --git a/lib/public/TextToImage/Exception/Exception.php b/lib/public/TextToImage/Exception/Exception.php new file mode 100644 index 0000000000000..106748b359ab9 --- /dev/null +++ b/lib/public/TextToImage/Exception/Exception.php @@ -0,0 +1,29 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\TextToImage\Exception; + +class Exception extends \Exception { + +} diff --git a/lib/public/TextToImage/Exception/TaskNotFoundException.php b/lib/public/TextToImage/Exception/TaskNotFoundException.php new file mode 100644 index 0000000000000..eef9e113a2b7b --- /dev/null +++ b/lib/public/TextToImage/Exception/TaskNotFoundException.php @@ -0,0 +1,29 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\TextToImage\Exception; + +class TaskNotFoundException extends Exception { + +} diff --git a/lib/public/TextToImage/IManager.php b/lib/public/TextToImage/IManager.php new file mode 100644 index 0000000000000..e24e6ffd3a010 --- /dev/null +++ b/lib/public/TextToImage/IManager.php @@ -0,0 +1,98 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +namespace OCP\TextToImage; + +use OCP\PreConditionNotMetException; +use OCP\TextToImage\Exception\TaskNotFoundException; +use RuntimeException; + +/** + * API surface for apps interacting with and making use of TextToImage providers + * without knowing which providers are installed + * @since 28.0.0 + */ +interface IManager { + /** + * @since 28.0.0 + */ + public function hasProviders(): bool; + + /** + * @param Task $task The task to run + * @throws PreConditionNotMetException If no or not the requested provider was registered but this method was still called + * @throws RuntimeException If something else failed + * @since 28.0.0 + */ + public function runTask(Task $task): void; + + /** + * Will schedule a TextToImage process in the background. The result will become available + * with the \OCP\TextToImage\TaskSuccessfulEvent + * If inference fails a \OCP\TextToImage\Events\TaskFailedEvent will be dispatched instead + * + * @param Task $task The task to schedule + * @throws PreConditionNotMetException If no or not the requested provider was registered but this method was still called + * @since 28.0.0 + */ + public function scheduleTask(Task $task) : void; + + /** + * Delete a task that has been scheduled before + * + * @param Task $task The task to delete + * @since 28.0.0 + */ + public function deleteTask(Task $task): void; + + /** + * @param int $id The id of the task + * @return Task + * @throws RuntimeException If the query failed + * @throws TaskNotFoundException If the task could not be found + * @since 28.0.0 + */ + public function getTask(int $id): Task; + + /** + * @param int $id The id of the task + * @param string|null $userId The user id that scheduled the task + * @return Task + * @throws RuntimeException If the query failed + * @throws TaskNotFoundException If the task could not be found + * @since 28.0.0 + */ + public function getUserTask(int $id, ?string $userId): Task; + + /** + * @param string $userId + * @param string $appId + * @param string|null $identifier + * @return array + * @since 28.0.0 + */ + public function getUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array; +} diff --git a/lib/public/TextToImage/IProvider.php b/lib/public/TextToImage/IProvider.php new file mode 100644 index 0000000000000..4fc7308924308 --- /dev/null +++ b/lib/public/TextToImage/IProvider.php @@ -0,0 +1,52 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCP\TextToImage; + +use RuntimeException; + +/** + * This is the interface that is implemented by apps that + * implement a text to image provider + * @since 28.0.0 + */ +interface IProvider { + /** + * The localized name of this provider + * @since 28.0.0 + */ + public function getName(): string; + + /** + * Processes a text + * + * @param string $prompt The input text + * @param resource $resource The file resource to write the image to + * @return void + * @since 28.0.0 + * @throws RuntimeException If the text could not be processed + */ + public function generate(string $prompt, $resource): void; +} diff --git a/lib/public/TextToImage/Task.php b/lib/public/TextToImage/Task.php new file mode 100644 index 0000000000000..05ef1f5195f81 --- /dev/null +++ b/lib/public/TextToImage/Task.php @@ -0,0 +1,179 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCP\TextToImage; + +use OCP\IImage; +use OCP\Image; +use OCP\TextProcessing\IProvider as P; +use OCP\TextProcessing\ITaskType; + +/** + * This is a text to image task + * + * @since 28.0.0 + */ +final class Task implements \JsonSerializable { + protected ?int $id = null; + + private ?IImage $image = null; + + /** + * @since 28.0.0 + */ + public const STATUS_FAILED = 4; + /** + * @since 28.0.0 + */ + public const STATUS_SUCCESSFUL = 3; + /** + * @since 28.0.0 + */ + public const STATUS_RUNNING = 2; + /** + * @since 28.0.0 + */ + public const STATUS_SCHEDULED = 1; + /** + * @since 28.0.0 + */ + public const STATUS_UNKNOWN = 0; + + /** + * @psalm-var self::STATUS_* + */ + protected int $status = self::STATUS_UNKNOWN; + + /** + * @param string $input + * @param string $appId + * @param string|null $userId + * @param string $identifier An arbitrary identifier for this task. max length: 255 chars + * @since 28.0.0 + */ + final public function __construct( + protected string $input, + protected string $appId, + protected ?string $userId, + protected string $identifier = '', + ) { + } + + /** + * @return IImage|null + * @since 28.0.0 + */ + final public function getOutputImage(): ?IImage { + return $this->image; + } + + /** + * @param IImage|null $image + * @since 28.0.0 + */ + final public function setOutputImage(?IImage $image): void { + $this->image = $image; + } + + /** + * @psalm-return self::STATUS_* + * @since 28.0.0 + */ + final public function getStatus(): int { + return $this->status; + } + + /** + * @psalm-param self::STATUS_* $status + * @since 28.0.0 + */ + final public function setStatus(int $status): void { + $this->status = $status; + } + + /** + * @return int|null + * @since 28.0.0 + */ + final public function getId(): ?int { + return $this->id; + } + + /** + * @param int|null $id + * @since 28.0.0 + */ + final public function setId(?int $id): void { + $this->id = $id; + } + + /** + * @return string + * @since 28.0.0 + */ + final public function getInput(): string { + return $this->input; + } + + /** + * @return string + * @since 28.0.0 + */ + final public function getAppId(): string { + return $this->appId; + } + + /** + * @return string + * @since 28.0.0 + */ + final public function getIdentifier(): string { + return $this->identifier; + } + + /** + * @return string|null + * @since 28.0.0 + */ + final public function getUserId(): ?string { + return $this->userId; + } + + /** + * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, output: ?string, identifier: string} + * @since 28.0.0 + */ + public function jsonSerialize(): array { + return [ + 'id' => $this->getId(), + 'status' => $this->getStatus(), + 'userId' => $this->getUserId(), + 'appId' => $this->getAppId(), + 'input' => $this->getInput(), + 'result' => $this->getOutput(), + 'identifier' => $this->getIdentifier(), + ]; + } +} From 2d44c7c1ed4487b946a72eb9d32da59d799d8c66 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 7 Sep 2023 13:03:38 +0200 Subject: [PATCH 02/66] Small fixes Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 13 ++++++------- lib/public/TextToImage/Task.php | 3 +-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 4d2f6c81c90b2..93dab0a5d49b4 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -77,10 +77,9 @@ public function isAvailable(): DataResponse { * @param string $appId ID of the app that will execute the task * @param string $identifier An arbitrary identifier for the task * - * @return DataResponse|DataResponse + * @return DataResponse|DataResponse * * 200: Task scheduled successfully - * 400: Scheduling task is not possible * 412: Scheduling task is not possible */ #[PublicPage] @@ -122,9 +121,9 @@ public function getTask(int $id): DataResponse { return new DataResponse([ 'task' => $json, ]); - } catch (TaskNotFoundException $e) { + } catch (TaskNotFoundException) { return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); - } catch (\RuntimeException $e) { + } catch (\RuntimeException) { return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); } } @@ -184,9 +183,9 @@ public function deleteTask(int $id): DataResponse { return new DataResponse([ 'task' => $json, ]); - } catch (TaskNotFoundException $e) { + } catch (TaskNotFoundException) { return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); - } catch (\RuntimeException $e) { + } catch (\RuntimeException) { return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); } } @@ -214,7 +213,7 @@ public function listTasksByApp(string $appId, ?string $identifier = null): DataR return new DataResponse([ 'tasks' => $json, ]); - } catch (\RuntimeException $e) { + } catch (\RuntimeException) { return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); } } diff --git a/lib/public/TextToImage/Task.php b/lib/public/TextToImage/Task.php index 05ef1f5195f81..2e839389a973f 100644 --- a/lib/public/TextToImage/Task.php +++ b/lib/public/TextToImage/Task.php @@ -162,7 +162,7 @@ final public function getUserId(): ?string { } /** - * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, output: ?string, identifier: string} + * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, identifier: string} * @since 28.0.0 */ public function jsonSerialize(): array { @@ -172,7 +172,6 @@ public function jsonSerialize(): array { 'userId' => $this->getUserId(), 'appId' => $this->getAppId(), 'input' => $this->getInput(), - 'result' => $this->getOutput(), 'identifier' => $this->getIdentifier(), ]; } From 666f7b33c9a291d0222dbcfcf520f223b84c9301 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 7 Sep 2023 15:01:56 +0200 Subject: [PATCH 03/66] Small fixes Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Manager.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index 8700ea8b5675d..6333292a5034d 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -130,7 +130,6 @@ public function runTask(Task $task): void { } $file = $folder->newFile((string) $task->getId()); $provider->generate($task->getInput(), $file->write()); - $task->setResultPath($file->getName()); $task->setStatus(Task::STATUS_SUCCESSFUL); $this->taskMapper->update(DbTask::fromPublicTask($task)); return; @@ -184,7 +183,7 @@ public function deleteTask(Task $task): void { * @param int $id The id of the task * @return Task * @throws RuntimeException If the query failed - * @throws NotFoundException If the task could not be found + * @throws TaskNotFoundException If the task could not be found */ public function getTask(int $id): Task { try { @@ -207,7 +206,7 @@ public function getTask(int $id): Task { * @param string|null $userId The user id that scheduled the task * @return Task * @throws RuntimeException If the query failed - * @throws NotFoundException If the task could not be found + * @throws TaskNotFoundException If the task could not be found */ public function getUserTask(int $id, ?string $userId): Task { try { From dc8cba6026febee057c08012fc2cbc6e55f1fc91 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 8 Sep 2023 12:03:52 +0200 Subject: [PATCH 04/66] cs:Fix Signed-off-by: Marcel Klehr --- lib/public/TextToImage/Exception/Exception.php | 1 - lib/public/TextToImage/Exception/TaskNotFoundException.php | 1 - lib/public/TextToImage/Task.php | 3 --- 3 files changed, 5 deletions(-) diff --git a/lib/public/TextToImage/Exception/Exception.php b/lib/public/TextToImage/Exception/Exception.php index 106748b359ab9..5e0e7ab1fdac8 100644 --- a/lib/public/TextToImage/Exception/Exception.php +++ b/lib/public/TextToImage/Exception/Exception.php @@ -25,5 +25,4 @@ namespace OCP\TextToImage\Exception; class Exception extends \Exception { - } diff --git a/lib/public/TextToImage/Exception/TaskNotFoundException.php b/lib/public/TextToImage/Exception/TaskNotFoundException.php index eef9e113a2b7b..abdd8e1526158 100644 --- a/lib/public/TextToImage/Exception/TaskNotFoundException.php +++ b/lib/public/TextToImage/Exception/TaskNotFoundException.php @@ -25,5 +25,4 @@ namespace OCP\TextToImage\Exception; class TaskNotFoundException extends Exception { - } diff --git a/lib/public/TextToImage/Task.php b/lib/public/TextToImage/Task.php index 2e839389a973f..81a33752262bb 100644 --- a/lib/public/TextToImage/Task.php +++ b/lib/public/TextToImage/Task.php @@ -26,9 +26,6 @@ namespace OCP\TextToImage; use OCP\IImage; -use OCP\Image; -use OCP\TextProcessing\IProvider as P; -use OCP\TextProcessing\ITaskType; /** * This is a text to image task From 1e36d74c3ebb595cfaeb337198418b0769cab521 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 16 Oct 2023 15:41:56 +0200 Subject: [PATCH 05/66] Update core/Controller/TextToImageApiController.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julius Härtl Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 93dab0a5d49b4..6ec715547fea4 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -47,12 +47,12 @@ */ class TextToImageApiController extends \OCP\AppFramework\OCSController { public function __construct( - string $appName, - IRequest $request, - private IManager $textToImageManager, - private IL10N $l, - private ?string $userId, - private AppData $appData, + string $appName, + IRequest $request, + private IManager $textToImageManager, + private IL10N $l, + private ?string $userId, + private AppData $appData, ) { parent::__construct($appName, $request); } From e199d1aae1dbaa53651d8b715fd5609109e9f3f6 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 16 Oct 2023 15:43:27 +0200 Subject: [PATCH 06/66] Update lib/public/TextToImage/IManager.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julius Härtl Signed-off-by: Marcel Klehr --- lib/public/TextToImage/IManager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/public/TextToImage/IManager.php b/lib/public/TextToImage/IManager.php index e24e6ffd3a010..dc7488156bc4e 100644 --- a/lib/public/TextToImage/IManager.php +++ b/lib/public/TextToImage/IManager.php @@ -91,7 +91,7 @@ public function getUserTask(int $id, ?string $userId): Task; * @param string $userId * @param string $appId * @param string|null $identifier - * @return array + * @return Task[] * @since 28.0.0 */ public function getUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array; From 53678a853443f615a8fcfb01321d9e1416ada5b4 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 16 Oct 2023 15:58:33 +0200 Subject: [PATCH 07/66] Update lib/public/TextToImage/Events/TaskSuccessfulEvent.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julius Härtl Signed-off-by: Marcel Klehr --- lib/public/TextToImage/Events/TaskSuccessfulEvent.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/public/TextToImage/Events/TaskSuccessfulEvent.php b/lib/public/TextToImage/Events/TaskSuccessfulEvent.php index 3e2e76198daa5..15d263c0ff0e7 100644 --- a/lib/public/TextToImage/Events/TaskSuccessfulEvent.php +++ b/lib/public/TextToImage/Events/TaskSuccessfulEvent.php @@ -27,7 +27,7 @@ namespace OCP\TextToImage\Events; /** - * @since 27.1.0 + * @since 28.0.0 */ class TaskSuccessfulEvent extends AbstractTextToImageEvent { } From 41847c951a48cc2c79992656b6a5bf59d9334e1f Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 16 Oct 2023 15:58:53 +0200 Subject: [PATCH 08/66] Update lib/private/TextToImage/TaskBackgroundJob.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Julius Härtl Signed-off-by: Marcel Klehr --- lib/private/TextToImage/TaskBackgroundJob.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/private/TextToImage/TaskBackgroundJob.php b/lib/private/TextToImage/TaskBackgroundJob.php index b223258485a84..ac5cd6b59b5f1 100644 --- a/lib/private/TextToImage/TaskBackgroundJob.php +++ b/lib/private/TextToImage/TaskBackgroundJob.php @@ -35,8 +35,8 @@ class TaskBackgroundJob extends QueuedJob { public function __construct( - ITimeFactory $timeFactory, - private IManager $text2imageManager, + ITimeFactory $timeFactory, + private IManager $text2imageManager, private IEventDispatcher $eventDispatcher, ) { parent::__construct($timeFactory); From e5efbc88d864386a441ee8f2534888652cdcf4b0 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 16 Oct 2023 16:19:19 +0200 Subject: [PATCH 09/66] enh(TextToImage): Address review comments Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 5 +- .../Version28000Date20230906104802.php | 10 +-- lib/private/TextToImage/Manager.php | 64 +++++++++++++++---- lib/public/DB/Exception.php | 2 +- .../Exception/TaskNotFoundException.php | 2 +- ...Exception.php => TextToImageException.php} | 2 +- 6 files changed, 56 insertions(+), 29 deletions(-) rename lib/public/TextToImage/Exception/{Exception.php => TextToImageException.php} (94%) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 6ec715547fea4..02692f09cdf35 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -58,12 +58,11 @@ public function __construct( } /** - * @PublicPage - * - * Check whether this feature is available + * * Check whether this feature is available * * @return DataResponse */ + #[PublicPage] public function isAvailable(): DataResponse { return new DataResponse([ 'isAvailable' => $this->textToImageManager->hasProviders(), diff --git a/core/Migrations/Version28000Date20230906104802.php b/core/Migrations/Version28000Date20230906104802.php index 662bdd648b7d3..61f0d01dff0e4 100644 --- a/core/Migrations/Version28000Date20230906104802.php +++ b/core/Migrations/Version28000Date20230906104802.php @@ -45,7 +45,6 @@ class Version28000Date20230906104802 extends SimpleMigrationStep { public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { /** @var ISchemaWrapper $schema */ $schema = $schemaClosure(); - $changed = false; if (!$schema->hasTable('text2image_tasks')) { $table = $schema->createTable('text2image_tasks'); @@ -76,11 +75,8 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt 'length' => 255, 'default' => '', ]); - $table->addColumn('last_updated', Types::INTEGER, [ + $table->addColumn('last_updated', Types::DATETIME, [ 'notnull' => false, - 'length' => 4, - 'default' => 0, - 'unsigned' => true, ]); $table->setPrimaryKey(['id'], 't2i_tasks_id_index'); @@ -88,10 +84,6 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table->addIndex(['status'], 't2i_tasks_status'); $table->addIndex(['user_id', 'app_id', 'identifier'], 't2i_tasks_uid_appid_ident'); - $changed = true; - } - - if ($changed) { return $schema; } diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index 6333292a5034d..fd9e2212abda5 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -29,6 +29,8 @@ use OC\TextToImage\Db\Task as DbTask; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; use OCP\IConfig; use OCP\TextToImage\Exception\TaskNotFoundException; use OCP\TextToImage\IManager; @@ -100,6 +102,7 @@ public function hasProviders(): bool { * @inheritDoc */ public function runTask(Task $task): void { + $this->logger->debug('Running TextToImage Task'); if (!$this->hasProviders()) { throw new PreConditionNotMetException('No text to image provider is installed that can handle this task'); } @@ -107,42 +110,75 @@ public function runTask(Task $task): void { $json = $this->config->getAppValue('core', 'ai.text2image_provider', ''); if ($json !== '') { - $className = json_decode($json, true); - $provider = current(array_filter($providers, fn ($provider) => $provider::class === $className)); - if ($provider !== false) { - $providers = [$provider]; + try { + $className = json_decode($json, true, 512, JSON_THROW_ON_ERROR); + $provider = current(array_filter($providers, fn ($provider) => $provider::class === $className)); + if ($provider !== false) { + $providers = [$provider]; + } + } catch (\JsonException $e) { + $this->logger->warning('Failed to decode Text2Image setting `ai.text2image_provider`', ['exception' => $e]); } } foreach ($providers as $provider) { + $this->logger->debug('Trying to run Text2Image provider '.$provider::class); try { $task->setStatus(Task::STATUS_RUNNING); if ($task->getId() === null) { + $this->logger->debug('Inserting Text2Image task into DB'); $taskEntity = $this->taskMapper->insert(DbTask::fromPublicTask($task)); $task->setId($taskEntity->getId()); } else { + $this->logger->debug('Updating Text2Image task in DB'); $this->taskMapper->update(DbTask::fromPublicTask($task)); } try { $folder = $this->appData->getFolder('text2image'); - } catch(\OCP\Files\NotFoundException $e) { + } catch(NotFoundException) { + $this->logger->debug('Creating folder in appdata for Text2Image results'); $folder = $this->appData->newFolder('text2image'); } + $this->logger->debug('Creating result file for Text2Image task'); $file = $folder->newFile((string) $task->getId()); - $provider->generate($task->getInput(), $file->write()); + $resource = $file->write(); + if ($resource === false) { + throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: Couldn\'t open file to write.'); + } + $this->logger->debug('Calling Text2Image provider\'s generate method'); + $provider->generate($task->getInput(), $resource); + if (is_resource($resource)) { + // If $resource hasn't been closed yet, we'll do that here + fclose($resource); + } $task->setStatus(Task::STATUS_SUCCESSFUL); + $this->logger->debug('Updating Text2Image task in DB'); $this->taskMapper->update(DbTask::fromPublicTask($task)); return; - } catch (\RuntimeException $e) { - $this->logger->info('Text2Image generation using provider ' . $provider->getName() . ' failed', ['exception' => $e]); - $task->setStatus(Task::STATUS_FAILED); - $this->taskMapper->update(DbTask::fromPublicTask($task)); - throw $e; - } catch (\Throwable $e) { + } catch (\RuntimeException|\Throwable $e) { + if (isset($resource) && is_resource($resource)) { + // If $resource hasn't been closed yet, we'll do that here + fclose($resource); + } + try { + if (isset($file)) { + $file->delete(); + } + }catch(NotPermittedException $e) { + $this->logger->warning('Failed to clean up Text2Image result file after error', ['exception' => $e]); + } $this->logger->info('Text2Image generation using provider ' . $provider->getName() . ' failed', ['exception' => $e]); $task->setStatus(Task::STATUS_FAILED); - $this->taskMapper->update(DbTask::fromPublicTask($task)); - throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: ' . $e->getMessage(), 0, $e); + try { + $this->taskMapper->update(DbTask::fromPublicTask($task)); + } catch (Exception $e) { + $this->logger->warning('Failed to update database after Text2Image error', ['exception' => $e]); + } + if ($e instanceof RuntimeException) { + throw $e; + }else { + throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: ' . $e->getMessage(), 0, $e); + } } } diff --git a/lib/public/DB/Exception.php b/lib/public/DB/Exception.php index f977ffa739630..987666b05a96e 100644 --- a/lib/public/DB/Exception.php +++ b/lib/public/DB/Exception.php @@ -139,7 +139,7 @@ class Exception extends BaseException { /** * @return int|null - * @psalm-return Exception::REASON_* + * @psalm-return TextToImageException::REASON_* * @since 21.0.0 */ public function getReason(): ?int { diff --git a/lib/public/TextToImage/Exception/TaskNotFoundException.php b/lib/public/TextToImage/Exception/TaskNotFoundException.php index abdd8e1526158..7bb81d0114611 100644 --- a/lib/public/TextToImage/Exception/TaskNotFoundException.php +++ b/lib/public/TextToImage/Exception/TaskNotFoundException.php @@ -24,5 +24,5 @@ namespace OCP\TextToImage\Exception; -class TaskNotFoundException extends Exception { +class TaskNotFoundException extends TextToImageException { } diff --git a/lib/public/TextToImage/Exception/Exception.php b/lib/public/TextToImage/Exception/TextToImageException.php similarity index 94% rename from lib/public/TextToImage/Exception/Exception.php rename to lib/public/TextToImage/Exception/TextToImageException.php index 5e0e7ab1fdac8..59322bd44a4a7 100644 --- a/lib/public/TextToImage/Exception/Exception.php +++ b/lib/public/TextToImage/Exception/TextToImageException.php @@ -24,5 +24,5 @@ namespace OCP\TextToImage\Exception; -class Exception extends \Exception { +class TextToImageException extends \Exception { } From 3e6a8b31c4eb7792e20315b96db86856c0f51bb6 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 16 Oct 2023 16:33:07 +0200 Subject: [PATCH 10/66] fix(TextToImage): Fix psalm errors Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Manager.php | 2 +- lib/public/DB/Exception.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index fd9e2212abda5..8e56558d9c3a1 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -265,7 +265,7 @@ public function getUserTask(int $id, ?string $userId): Task { * @param string $userId * @param string $appId * @param string|null $identifier - * @return array + * @return Task[] */ public function getUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array { try { diff --git a/lib/public/DB/Exception.php b/lib/public/DB/Exception.php index 987666b05a96e..f977ffa739630 100644 --- a/lib/public/DB/Exception.php +++ b/lib/public/DB/Exception.php @@ -139,7 +139,7 @@ class Exception extends BaseException { /** * @return int|null - * @psalm-return TextToImageException::REASON_* + * @psalm-return Exception::REASON_* * @since 21.0.0 */ public function getReason(): ?int { From 207c95838f16216382c9c4fc0e094c1db5a49299 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 16 Oct 2023 16:35:11 +0200 Subject: [PATCH 11/66] fix(TextToImage): Fix coding style Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Manager.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index 8e56558d9c3a1..f6828cda59525 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -164,7 +164,7 @@ public function runTask(Task $task): void { if (isset($file)) { $file->delete(); } - }catch(NotPermittedException $e) { + } catch(NotPermittedException $e) { $this->logger->warning('Failed to clean up Text2Image result file after error', ['exception' => $e]); } $this->logger->info('Text2Image generation using provider ' . $provider->getName() . ' failed', ['exception' => $e]); @@ -176,7 +176,7 @@ public function runTask(Task $task): void { } if ($e instanceof RuntimeException) { throw $e; - }else { + } else { throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: ' . $e->getMessage(), 0, $e); } } From c59861a9fa1f0bc6fe6958abe99da9240af0d092 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 17 Oct 2023 11:54:01 +0200 Subject: [PATCH 12/66] enh(TextToImage): Implement removal of stale images and change Task#last_updated to DATETIME Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Db/Task.php | 7 +++--- lib/private/TextToImage/Db/TaskMapper.php | 20 ++++++++++++---- .../RemoveOldTasksBackgroundJob.php | 23 +++++++++++++++++-- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/lib/private/TextToImage/Db/Task.php b/lib/private/TextToImage/Db/Task.php index 84d43ab6fd999..e545f459d6c68 100644 --- a/lib/private/TextToImage/Db/Task.php +++ b/lib/private/TextToImage/Db/Task.php @@ -25,6 +25,7 @@ namespace OC\TextToImage\Db; +use DateTime; use OCP\AppFramework\Db\Entity; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\NotFoundException; @@ -33,8 +34,8 @@ use OCP\TextToImage\Task as OCPTask; /** - * @method setLastUpdated(int $lastUpdated) - * @method int getLastUpdated() + * @method setLastUpdated(DateTime $lastUpdated) + * @method DateTime getLastUpdated() * @method setInput(string $type) * @method string getInput() * @method setResultPath(string $resultPath) @@ -71,7 +72,7 @@ class Task extends Entity { public function __construct() { // add types in constructor $this->addType('id', 'integer'); - $this->addType('lastUpdated', 'integer'); + $this->addType('lastUpdated', 'datetime'); $this->addType('input', 'string'); $this->addType('status', 'integer'); $this->addType('userId', 'string'); diff --git a/lib/private/TextToImage/Db/TaskMapper.php b/lib/private/TextToImage/Db/TaskMapper.php index 44d8aea24eb4a..f65d5a1d17bf6 100644 --- a/lib/private/TextToImage/Db/TaskMapper.php +++ b/lib/private/TextToImage/Db/TaskMapper.php @@ -25,12 +25,14 @@ namespace OC\TextToImage\Db; +use DateTime; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Db\QBMapper; use OCP\AppFramework\Utility\ITimeFactory; use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\IDBConnection; /** @@ -101,18 +103,26 @@ public function findUserTasksByApp(string $userId, string $appId, ?string $ident /** * @param int $timeout - * @return int the number of deleted tasks + * @return Task[] the deleted tasks * @throws Exception */ - public function deleteOlderThan(int $timeout): int { + public function deleteOlderThan(int $timeout): array { + $datetime = new DateTime(); + $datetime->sub(new \DateInterval('PT'.$timeout.'S')); + $qb = $this->db->getQueryBuilder(); + $qb->select('*') + ->from($this->tableName) + ->where($qb->expr()->lt('last_updated', $qb->createPositionalParameter($datetime, IQueryBuilder::PARAM_DATE))); + $deletedTasks = $this->findEntities($qb); $qb = $this->db->getQueryBuilder(); $qb->delete($this->tableName) - ->where($qb->expr()->lt('last_updated', $qb->createPositionalParameter(time() - $timeout))); - return $qb->executeStatement(); + ->where($qb->expr()->lt('last_updated', $qb->createPositionalParameter($datetime, IQueryBuilder::PARAM_DATE))); + $qb->executeStatement(); + return $deletedTasks; } public function update(Entity $entity): Entity { - $entity->setLastUpdated($this->timeFactory->now()->getTimestamp()); + $entity->setLastUpdated(DateTime::createFromImmutable($this->timeFactory->now())); return parent::update($entity); } } diff --git a/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php b/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php index fe6c77cb7902e..82677fa635c6c 100644 --- a/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php +++ b/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php @@ -30,18 +30,25 @@ use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\Exception; +use OCP\Files\AppData\IAppDataFactory; +use OCP\Files\IAppData; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; use Psr\Log\LoggerInterface; class RemoveOldTasksBackgroundJob extends TimedJob { public const MAX_TASK_AGE_SECONDS = 60 * 50 * 24 * 7; // 1 week + private IAppData $appData; + public function __construct( ITimeFactory $timeFactory, private TaskMapper $taskMapper, private LoggerInterface $logger, - + private IAppDataFactory $appDataFactory, ) { parent::__construct($timeFactory); + $this->appData = $this->appDataFactory->get('core'); $this->setInterval(60 * 60 * 24); } @@ -51,9 +58,21 @@ public function __construct( */ protected function run($argument) { try { - $this->taskMapper->deleteOlderThan(self::MAX_TASK_AGE_SECONDS); + $deletedTasks = $this->taskMapper->deleteOlderThan(self::MAX_TASK_AGE_SECONDS); + $folder = $this->appData->getFolder('text2image'); + foreach ($deletedTasks as $deletedTask) { + try { + $folder->getFile((string)$deletedTask->getId())->delete(); + } catch (NotFoundException) { + // noop + } catch (NotPermittedException $e) { + $this->logger->warning('Failed to delete stale text to image task', ['exception' => $e]); + } + } } catch (Exception $e) { $this->logger->warning('Failed to delete stale text to image tasks', ['exception' => $e]); + } catch(NotFoundException) { + // noop } } } From f52d763f50d10b8586c083919b287cc9a363f81e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 17 Oct 2023 12:23:02 +0200 Subject: [PATCH 13/66] enh(TextToImage): Add routes Signed-off-by: Marcel Klehr --- core/routes.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/routes.php b/core/routes.php index fcb5a15cf0120..a779b130ba25f 100644 --- a/core/routes.php +++ b/core/routes.php @@ -155,6 +155,13 @@ ['root' => '/textprocessing', 'name' => 'TextProcessingApi#getTask', 'url' => '/task/{id}', 'verb' => 'GET'], ['root' => '/textprocessing', 'name' => 'TextProcessingApi#deleteTask', 'url' => '/task/{id}', 'verb' => 'DELETE'], ['root' => '/textprocessing', 'name' => 'TextProcessingApi#listTasksByApp', 'url' => '/tasks/app/{appId}', 'verb' => 'GET'], + + ['root' => '/text2image', 'name' => 'TextToImageApi#isAvailable', 'url' => '/is_available', 'verb' => 'GET'], + ['root' => '/text2image', 'name' => 'TextToImageApi#schedule', 'url' => '/schedule', 'verb' => 'POST'], + ['root' => '/text2image', 'name' => 'TextToImageApi#getTask', 'url' => '/task/{id}', 'verb' => 'GET'], + ['root' => '/text2image', 'name' => 'TextToImageApi#getImage', 'url' => '/task/{id}/image', 'verb' => 'GET'], + ['root' => '/text2image', 'name' => 'TextToImageApi#deleteTask', 'url' => '/task/{id}', 'verb' => 'DELETE'], + ['root' => '/text2image', 'name' => 'TextToImageApi#listTasksByApp', 'url' => '/tasks/app/{appId}', 'verb' => 'GET'], ], ]); From e8faaebb0fc2e57cb9191529c356b33ebe454a4a Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 17 Oct 2023 16:53:42 +0200 Subject: [PATCH 14/66] enh(TextToImage): Allow anonymous access to IManager#getUserTasksByApp Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Db/TaskMapper.php | 2 +- lib/private/TextToImage/Manager.php | 2 +- lib/public/TextToImage/IManager.php | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/private/TextToImage/Db/TaskMapper.php b/lib/private/TextToImage/Db/TaskMapper.php index f65d5a1d17bf6..2cf9cff588260 100644 --- a/lib/private/TextToImage/Db/TaskMapper.php +++ b/lib/private/TextToImage/Db/TaskMapper.php @@ -89,7 +89,7 @@ public function findByIdAndUser(int $id, ?string $userId): Task { * @return array * @throws Exception */ - public function findUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array { + public function findUserTasksByApp(?string $userId, string $appId, ?string $identifier = null): array { $qb = $this->db->getQueryBuilder(); $qb->select(Task::$columns) ->from($this->tableName) diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index f6828cda59525..192a5e5dc9826 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -267,7 +267,7 @@ public function getUserTask(int $id, ?string $userId): Task { * @param string|null $identifier * @return Task[] */ - public function getUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array { + public function getUserTasksByApp(?string $userId, string $appId, ?string $identifier = null): array { try { $taskEntities = $this->taskMapper->findUserTasksByApp($userId, $appId, $identifier); return array_map(static function (DbTask $taskEntity) { diff --git a/lib/public/TextToImage/IManager.php b/lib/public/TextToImage/IManager.php index dc7488156bc4e..53141f3ce394a 100644 --- a/lib/public/TextToImage/IManager.php +++ b/lib/public/TextToImage/IManager.php @@ -88,11 +88,11 @@ public function getTask(int $id): Task; public function getUserTask(int $id, ?string $userId): Task; /** - * @param string $userId + * @param ?string $userId * @param string $appId * @param string|null $identifier * @return Task[] * @since 28.0.0 */ - public function getUserTasksByApp(string $userId, string $appId, ?string $identifier = null): array; + public function getUserTasksByApp(?string $userId, string $appId, ?string $identifier = null): array; } From 1d07dcc346f8e398786093d48f656ca0128334ad Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 10:55:57 +0200 Subject: [PATCH 15/66] Update lib/private/TextToImage/Db/Task.php Co-authored-by: Joas Schilling <213943+nickvergessen@users.noreply.github.com> Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Db/Task.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/TextToImage/Db/Task.php b/lib/private/TextToImage/Db/Task.php index e545f459d6c68..bff827533a59d 100644 --- a/lib/private/TextToImage/Db/Task.php +++ b/lib/private/TextToImage/Db/Task.php @@ -47,7 +47,7 @@ * @method setAppId(string $type) * @method string getAppId() * @method setIdentifier(string $identifier) - * @method string getIdentifier() + * @method string|null getIdentifier() */ class Task extends Entity { protected $lastUpdated; From 47372380b479028087f4026f09bb8d0cdc3f5e24 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 10:58:49 +0200 Subject: [PATCH 16/66] Update lib/public/TextToImage/Exception/TaskNotFoundException.php Co-authored-by: Joas Schilling <213943+nickvergessen@users.noreply.github.com> Signed-off-by: Marcel Klehr --- lib/public/TextToImage/Exception/TaskNotFoundException.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/public/TextToImage/Exception/TaskNotFoundException.php b/lib/public/TextToImage/Exception/TaskNotFoundException.php index 7bb81d0114611..bd713fe3905ca 100644 --- a/lib/public/TextToImage/Exception/TaskNotFoundException.php +++ b/lib/public/TextToImage/Exception/TaskNotFoundException.php @@ -24,5 +24,8 @@ namespace OCP\TextToImage\Exception; +/** + * @since 28.0.0 + */ class TaskNotFoundException extends TextToImageException { } From 5ddf3c336604a369461913b082369f7729be5760 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 10:59:13 +0200 Subject: [PATCH 17/66] Update lib/public/TextToImage/Exception/TextToImageException.php Co-authored-by: Joas Schilling <213943+nickvergessen@users.noreply.github.com> Signed-off-by: Marcel Klehr --- lib/public/TextToImage/Exception/TextToImageException.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/public/TextToImage/Exception/TextToImageException.php b/lib/public/TextToImage/Exception/TextToImageException.php index 59322bd44a4a7..3d4fd192c942a 100644 --- a/lib/public/TextToImage/Exception/TextToImageException.php +++ b/lib/public/TextToImage/Exception/TextToImageException.php @@ -24,5 +24,8 @@ namespace OCP\TextToImage\Exception; +/** + * @since 28.0.0 + */ class TextToImageException extends \Exception { } From c5fbe5a7bc2ce6f808f1e604b9ba46980bd76908 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 10:51:28 +0200 Subject: [PATCH 18/66] enh(TextToImage): Add bruteforce protection for anonymous API usage Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 02692f09cdf35..7a5e81ebcbf30 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -35,6 +35,7 @@ use OCP\AppFramework\Http\Attribute\UserRateLimit; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\FileDisplayResponse; +use OCP\Files\NotFoundException; use OCP\IL10N; use OCP\IRequest; use OCP\TextToImage\Exception\TaskNotFoundException; @@ -111,6 +112,7 @@ public function schedule(string $input, string $type, string $appId, string $ide * 404: Task not found */ #[PublicPage] + #[AnonRateLimit(limit: 5, period: 120)] public function getTask(int $id): DataResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -139,12 +141,13 @@ public function getTask(int $id): DataResponse { * 404: Task not found */ #[PublicPage] + #[AnonRateLimit(limit: 5, period: 120)] public function getImage(int $id): DataResponse|FileDisplayResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); try { $folder = $this->appData->getFolder('text2image'); - } catch(\OCP\Files\NotFoundException) { + } catch(NotFoundException) { $folder = $this->appData->newFolder('text2image'); } $file = $folder->getFile((string)$task->getId()); @@ -155,7 +158,7 @@ public function getImage(int $id): DataResponse|FileDisplayResponse { return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); } catch (\RuntimeException) { return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); - } catch (\OCP\Files\NotFoundException) { + } catch (NotFoundException) { return new DataResponse(['message' => $this->l->t('Image not found')], Http::STATUS_NOT_FOUND); } } @@ -171,6 +174,7 @@ public function getImage(int $id): DataResponse|FileDisplayResponse { * 404: Task not found */ #[NoAdminRequired] + #[AnonRateLimit(limit: 5, period: 120)] public function deleteTask(int $id): DataResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -201,6 +205,7 @@ public function deleteTask(int $id): DataResponse { * 200: Task list returned */ #[NoAdminRequired] + #[AnonRateLimit(limit: 5, period: 120)] public function listTasksByApp(string $appId, ?string $identifier = null): DataResponse { try { $tasks = $this->textToImageManager->getUserTasksByApp($this->userId, $appId, $identifier); From d3da49de4433f34c4cba23746a4f414309a40370 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 10:53:06 +0200 Subject: [PATCH 19/66] fix(TextToImage): Fix docblock of getImage route Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 7a5e81ebcbf30..8db31f4b659e1 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -130,15 +130,14 @@ public function getTask(int $id): DataResponse { } /** - * This endpoint allows checking the status and results of a task. - * Tasks are removed 1 week after receiving their last update. + * This endpoint allows downloading the resulting image of a task * * @param int $id The id of the task * * @return FileDisplayResponse|DataResponse * - * 200: Task returned - * 404: Task not found + * 200: Image returned + * 404: Task or image not found */ #[PublicPage] #[AnonRateLimit(limit: 5, period: 120)] From ca9a28ab51d9f64cd0802ecab5cbdcccfa8eacb9 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 10:55:30 +0200 Subject: [PATCH 20/66] fix(TextToImage): Fix notnull column to allow for empty strings on orcale Signed-off-by: Marcel Klehr --- core/Migrations/Version28000Date20230906104802.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Migrations/Version28000Date20230906104802.php b/core/Migrations/Version28000Date20230906104802.php index 61f0d01dff0e4..7134899c82e6b 100644 --- a/core/Migrations/Version28000Date20230906104802.php +++ b/core/Migrations/Version28000Date20230906104802.php @@ -71,7 +71,7 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt 'default' => '', ]); $table->addColumn('identifier', Types::STRING, [ - 'notnull' => true, + 'notnull' => false, 'length' => 255, 'default' => '', ]); From 9b7f63946f090f27728e58f8c809f50a27478859 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 13:33:04 +0200 Subject: [PATCH 21/66] fix(TextToImage): Fix psalm issues Signed-off-by: Marcel Klehr --- core/ResponseDefinitions.php | 2 +- lib/public/TextToImage/Task.php | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/ResponseDefinitions.php b/core/ResponseDefinitions.php index 97c91c5bbe0d2..2548880395aec 100644 --- a/core/ResponseDefinitions.php +++ b/core/ResponseDefinitions.php @@ -151,7 +151,7 @@ * userId: ?string, * appId: string, * input: string, - * identifier: string, + * identifier: ?string, * } */ class ResponseDefinitions { diff --git a/lib/public/TextToImage/Task.php b/lib/public/TextToImage/Task.php index 81a33752262bb..545bd8bac5a02 100644 --- a/lib/public/TextToImage/Task.php +++ b/lib/public/TextToImage/Task.php @@ -67,14 +67,14 @@ final class Task implements \JsonSerializable { * @param string $input * @param string $appId * @param string|null $userId - * @param string $identifier An arbitrary identifier for this task. max length: 255 chars + * @param null|string $identifier An arbitrary identifier for this task. max length: 255 chars * @since 28.0.0 */ final public function __construct( protected string $input, protected string $appId, protected ?string $userId, - protected string $identifier = '', + protected ?string $identifier = '', ) { } @@ -143,10 +143,10 @@ final public function getAppId(): string { } /** - * @return string + * @return null|string * @since 28.0.0 */ - final public function getIdentifier(): string { + final public function getIdentifier(): ?string { return $this->identifier; } @@ -159,7 +159,7 @@ final public function getUserId(): ?string { } /** - * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, identifier: string} + * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, identifier: ?string} * @since 28.0.0 */ public function jsonSerialize(): array { From ab856a5c782ffcc403fd5d4001eb52c570bd3195 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 14:09:04 +0200 Subject: [PATCH 22/66] fix(TextToImage): Fix psalm issues Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Manager.php | 1 + lib/public/TextToImage/IManager.php | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index 192a5e5dc9826..181748caf9979 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -266,6 +266,7 @@ public function getUserTask(int $id, ?string $userId): Task { * @param string $appId * @param string|null $identifier * @return Task[] + * @throws RuntimeException */ public function getUserTasksByApp(?string $userId, string $appId, ?string $identifier = null): array { try { diff --git a/lib/public/TextToImage/IManager.php b/lib/public/TextToImage/IManager.php index 53141f3ce394a..c5c8a0a22a60e 100644 --- a/lib/public/TextToImage/IManager.php +++ b/lib/public/TextToImage/IManager.php @@ -93,6 +93,7 @@ public function getUserTask(int $id, ?string $userId): Task; * @param string|null $identifier * @return Task[] * @since 28.0.0 + * @throws RuntimeException If the query failed */ public function getUserTasksByApp(?string $userId, string $appId, ?string $identifier = null): array; } From e57e94e11a2b25b114a8da28ca363bab23d3b12b Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 14:09:19 +0200 Subject: [PATCH 23/66] fix(TextToImage): Add bruteforce protection to API Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 38 ++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 8db31f4b659e1..08d9a6b5776df 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -30,6 +30,7 @@ use OCA\Core\ResponseDefinitions; use OCP\AppFramework\Http; use OCP\AppFramework\Http\Attribute\AnonRateLimit; +use OCP\AppFramework\Http\Attribute\BruteForceProtection; use OCP\AppFramework\Http\Attribute\NoAdminRequired; use OCP\AppFramework\Http\Attribute\PublicPage; use OCP\AppFramework\Http\Attribute\UserRateLimit; @@ -112,7 +113,8 @@ public function schedule(string $input, string $type, string $appId, string $ide * 404: Task not found */ #[PublicPage] - #[AnonRateLimit(limit: 5, period: 120)] + #[BruteForceProtection(action: 'not-found')] + #[BruteForceProtection(action: 'error')] public function getTask(int $id): DataResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -123,9 +125,13 @@ public function getTask(int $id): DataResponse { 'task' => $json, ]); } catch (TaskNotFoundException) { - return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + $res = new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + $res->throttle(['action' => 'not-found']); + return $res; } catch (\RuntimeException) { - return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + $res = new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + $res->throttle(['action' => 'error']); + return $res; } } @@ -140,7 +146,8 @@ public function getTask(int $id): DataResponse { * 404: Task or image not found */ #[PublicPage] - #[AnonRateLimit(limit: 5, period: 120)] + #[BruteForceProtection(action: 'not-found')] + #[BruteForceProtection(action: 'error')] public function getImage(int $id): DataResponse|FileDisplayResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -154,11 +161,17 @@ public function getImage(int $id): DataResponse|FileDisplayResponse { return new FileDisplayResponse($file, Http::STATUS_OK, ['Content-Type' => image_type_to_mime_type($info[2])]); } catch (TaskNotFoundException) { - return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + $res = new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + $res->throttle(['action' => 'not-found']); + return $res; } catch (\RuntimeException) { - return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + $res = new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + $res->throttle(['action' => 'error']); + return $res; } catch (NotFoundException) { - return new DataResponse(['message' => $this->l->t('Image not found')], Http::STATUS_NOT_FOUND); + $res = new DataResponse(['message' => $this->l->t('Image not found')], Http::STATUS_NOT_FOUND); + $res->throttle(['action' => 'not-found']); + return $res; } } @@ -173,7 +186,8 @@ public function getImage(int $id): DataResponse|FileDisplayResponse { * 404: Task not found */ #[NoAdminRequired] - #[AnonRateLimit(limit: 5, period: 120)] + #[BruteForceProtection(action: 'not-found')] + #[BruteForceProtection(action: 'error')] public function deleteTask(int $id): DataResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -186,9 +200,13 @@ public function deleteTask(int $id): DataResponse { 'task' => $json, ]); } catch (TaskNotFoundException) { - return new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + $res = new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); + $res->throttle(['action' => 'not-found']); + return $res; } catch (\RuntimeException) { - return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + $res = new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); + $res->throttle(['action' => 'error']); + return $res; } } From 6238aca6c50547b97a1c48cc897822601c114f15 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 18 Oct 2023 14:46:40 +0200 Subject: [PATCH 24/66] fix(TextToImage): Fix bruteforce protection Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 29 +++++++------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 08d9a6b5776df..921b3cbfb1178 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -113,8 +113,7 @@ public function schedule(string $input, string $type, string $appId, string $ide * 404: Task not found */ #[PublicPage] - #[BruteForceProtection(action: 'not-found')] - #[BruteForceProtection(action: 'error')] + #[BruteForceProtection(action: 'text2image')] public function getTask(int $id): DataResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -126,12 +125,10 @@ public function getTask(int $id): DataResponse { ]); } catch (TaskNotFoundException) { $res = new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); - $res->throttle(['action' => 'not-found']); + $res->throttle(['action' => 'text2image']); return $res; } catch (\RuntimeException) { - $res = new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); - $res->throttle(['action' => 'error']); - return $res; + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); } } @@ -146,8 +143,7 @@ public function getTask(int $id): DataResponse { * 404: Task or image not found */ #[PublicPage] - #[BruteForceProtection(action: 'not-found')] - #[BruteForceProtection(action: 'error')] + #[BruteForceProtection(action: 'text2image')] public function getImage(int $id): DataResponse|FileDisplayResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -162,15 +158,13 @@ public function getImage(int $id): DataResponse|FileDisplayResponse { return new FileDisplayResponse($file, Http::STATUS_OK, ['Content-Type' => image_type_to_mime_type($info[2])]); } catch (TaskNotFoundException) { $res = new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); - $res->throttle(['action' => 'not-found']); + $res->throttle(['action' => 'text2image']); return $res; } catch (\RuntimeException) { - $res = new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); - $res->throttle(['action' => 'error']); - return $res; + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); } catch (NotFoundException) { $res = new DataResponse(['message' => $this->l->t('Image not found')], Http::STATUS_NOT_FOUND); - $res->throttle(['action' => 'not-found']); + $res->throttle(['action' => 'text2image']); return $res; } } @@ -186,8 +180,7 @@ public function getImage(int $id): DataResponse|FileDisplayResponse { * 404: Task not found */ #[NoAdminRequired] - #[BruteForceProtection(action: 'not-found')] - #[BruteForceProtection(action: 'error')] + #[BruteForceProtection(action: 'text2image')] public function deleteTask(int $id): DataResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); @@ -201,12 +194,10 @@ public function deleteTask(int $id): DataResponse { ]); } catch (TaskNotFoundException) { $res = new DataResponse(['message' => $this->l->t('Task not found')], Http::STATUS_NOT_FOUND); - $res->throttle(['action' => 'not-found']); + $res->throttle(['action' => 'text2image']); return $res; } catch (\RuntimeException) { - $res = new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); - $res->throttle(['action' => 'error']); - return $res; + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); } } From 3d11ab7af83411f3245e1c467f6a4a5ce59e120e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 19 Oct 2023 10:54:18 +0200 Subject: [PATCH 25/66] fix(TextToImage): Add autoloader changes and registerAlias Signed-off-by: Marcel Klehr --- lib/composer/composer/autoload_classmap.php | 15 +++++++++++++++ lib/composer/composer/autoload_static.php | 15 +++++++++++++++ lib/private/Server.php | 2 ++ 3 files changed, 32 insertions(+) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 4387852556d96..f7a1db7716206 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -664,6 +664,14 @@ 'OCP\\TextProcessing\\SummaryTaskType' => $baseDir . '/lib/public/TextProcessing/SummaryTaskType.php', 'OCP\\TextProcessing\\Task' => $baseDir . '/lib/public/TextProcessing/Task.php', 'OCP\\TextProcessing\\TopicsTaskType' => $baseDir . '/lib/public/TextProcessing/TopicsTaskType.php', + 'OCP\\TextToImage\\Events\\AbstractTextToImageEvent' => $baseDir . '/lib/public/TextToImage/Events/AbstractTextToImageEvent.php', + 'OCP\\TextToImage\\Events\\TaskFailedEvent' => $baseDir . '/lib/public/TextToImage/Events/TaskFailedEvent.php', + 'OCP\\TextToImage\\Events\\TaskSuccessfulEvent' => $baseDir . '/lib/public/TextToImage/Events/TaskSuccessfulEvent.php', + 'OCP\\TextToImage\\Exception\\TaskNotFoundException' => $baseDir . '/lib/public/TextToImage/Exception/TaskNotFoundException.php', + 'OCP\\TextToImage\\Exception\\TextToImageException' => $baseDir . '/lib/public/TextToImage/Exception/TextToImageException.php', + 'OCP\\TextToImage\\IManager' => $baseDir . '/lib/public/TextToImage/IManager.php', + 'OCP\\TextToImage\\IProvider' => $baseDir . '/lib/public/TextToImage/IProvider.php', + 'OCP\\TextToImage\\Task' => $baseDir . '/lib/public/TextToImage/Task.php', 'OCP\\Translation\\CouldNotTranslateException' => $baseDir . '/lib/public/Translation/CouldNotTranslateException.php', 'OCP\\Translation\\IDetectLanguageProvider' => $baseDir . '/lib/public/Translation/IDetectLanguageProvider.php', 'OCP\\Translation\\ITranslationManager' => $baseDir . '/lib/public/Translation/ITranslationManager.php', @@ -1094,6 +1102,7 @@ 'OC\\Core\\Controller\\SearchController' => $baseDir . '/core/Controller/SearchController.php', 'OC\\Core\\Controller\\SetupController' => $baseDir . '/core/Controller/SetupController.php', 'OC\\Core\\Controller\\TextProcessingApiController' => $baseDir . '/core/Controller/TextProcessingApiController.php', + 'OC\\Core\\Controller\\TextToImageApiController' => $baseDir . '/core/Controller/TextToImageApiController.php', 'OC\\Core\\Controller\\TranslationApiController' => $baseDir . '/core/Controller/TranslationApiController.php', 'OC\\Core\\Controller\\TwoFactorChallengeController' => $baseDir . '/core/Controller/TwoFactorChallengeController.php', 'OC\\Core\\Controller\\UnifiedSearchController' => $baseDir . '/core/Controller/UnifiedSearchController.php', @@ -1175,6 +1184,7 @@ 'OC\\Core\\Migrations\\Version28000Date20230616104802' => $baseDir . '/core/Migrations/Version28000Date20230616104802.php', 'OC\\Core\\Migrations\\Version28000Date20230728104802' => $baseDir . '/core/Migrations/Version28000Date20230728104802.php', 'OC\\Core\\Migrations\\Version28000Date20230803221055' => $baseDir . '/core/Migrations/Version28000Date20230803221055.php', + 'OC\\Core\\Migrations\\Version28000Date20230906104802' => $baseDir . '/core/Migrations/Version28000Date20230906104802.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => $baseDir . '/lib/private/DB/Adapter.php', @@ -1710,6 +1720,11 @@ 'OC\\TextProcessing\\Manager' => $baseDir . '/lib/private/TextProcessing/Manager.php', 'OC\\TextProcessing\\RemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/TextProcessing/RemoveOldTasksBackgroundJob.php', 'OC\\TextProcessing\\TaskBackgroundJob' => $baseDir . '/lib/private/TextProcessing/TaskBackgroundJob.php', + 'OC\\TextToImage\\Db\\Task' => $baseDir . '/lib/private/TextToImage/Db/Task.php', + 'OC\\TextToImage\\Db\\TaskMapper' => $baseDir . '/lib/private/TextToImage/Db/TaskMapper.php', + 'OC\\TextToImage\\Manager' => $baseDir . '/lib/private/TextToImage/Manager.php', + 'OC\\TextToImage\\RemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php', + 'OC\\TextToImage\\TaskBackgroundJob' => $baseDir . '/lib/private/TextToImage/TaskBackgroundJob.php', 'OC\\Translation\\TranslationManager' => $baseDir . '/lib/private/Translation/TranslationManager.php', 'OC\\URLGenerator' => $baseDir . '/lib/private/URLGenerator.php', 'OC\\Updater' => $baseDir . '/lib/private/Updater.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 9b33577d66a26..70525168a0917 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -697,6 +697,14 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\TextProcessing\\SummaryTaskType' => __DIR__ . '/../../..' . '/lib/public/TextProcessing/SummaryTaskType.php', 'OCP\\TextProcessing\\Task' => __DIR__ . '/../../..' . '/lib/public/TextProcessing/Task.php', 'OCP\\TextProcessing\\TopicsTaskType' => __DIR__ . '/../../..' . '/lib/public/TextProcessing/TopicsTaskType.php', + 'OCP\\TextToImage\\Events\\AbstractTextToImageEvent' => __DIR__ . '/../../..' . '/lib/public/TextToImage/Events/AbstractTextToImageEvent.php', + 'OCP\\TextToImage\\Events\\TaskFailedEvent' => __DIR__ . '/../../..' . '/lib/public/TextToImage/Events/TaskFailedEvent.php', + 'OCP\\TextToImage\\Events\\TaskSuccessfulEvent' => __DIR__ . '/../../..' . '/lib/public/TextToImage/Events/TaskSuccessfulEvent.php', + 'OCP\\TextToImage\\Exception\\TaskNotFoundException' => __DIR__ . '/../../..' . '/lib/public/TextToImage/Exception/TaskNotFoundException.php', + 'OCP\\TextToImage\\Exception\\TextToImageException' => __DIR__ . '/../../..' . '/lib/public/TextToImage/Exception/TextToImageException.php', + 'OCP\\TextToImage\\IManager' => __DIR__ . '/../../..' . '/lib/public/TextToImage/IManager.php', + 'OCP\\TextToImage\\IProvider' => __DIR__ . '/../../..' . '/lib/public/TextToImage/IProvider.php', + 'OCP\\TextToImage\\Task' => __DIR__ . '/../../..' . '/lib/public/TextToImage/Task.php', 'OCP\\Translation\\CouldNotTranslateException' => __DIR__ . '/../../..' . '/lib/public/Translation/CouldNotTranslateException.php', 'OCP\\Translation\\IDetectLanguageProvider' => __DIR__ . '/../../..' . '/lib/public/Translation/IDetectLanguageProvider.php', 'OCP\\Translation\\ITranslationManager' => __DIR__ . '/../../..' . '/lib/public/Translation/ITranslationManager.php', @@ -1127,6 +1135,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Controller\\SearchController' => __DIR__ . '/../../..' . '/core/Controller/SearchController.php', 'OC\\Core\\Controller\\SetupController' => __DIR__ . '/../../..' . '/core/Controller/SetupController.php', 'OC\\Core\\Controller\\TextProcessingApiController' => __DIR__ . '/../../..' . '/core/Controller/TextProcessingApiController.php', + 'OC\\Core\\Controller\\TextToImageApiController' => __DIR__ . '/../../..' . '/core/Controller/TextToImageApiController.php', 'OC\\Core\\Controller\\TranslationApiController' => __DIR__ . '/../../..' . '/core/Controller/TranslationApiController.php', 'OC\\Core\\Controller\\TwoFactorChallengeController' => __DIR__ . '/../../..' . '/core/Controller/TwoFactorChallengeController.php', 'OC\\Core\\Controller\\UnifiedSearchController' => __DIR__ . '/../../..' . '/core/Controller/UnifiedSearchController.php', @@ -1208,6 +1217,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version28000Date20230616104802' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230616104802.php', 'OC\\Core\\Migrations\\Version28000Date20230728104802' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230728104802.php', 'OC\\Core\\Migrations\\Version28000Date20230803221055' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230803221055.php', + 'OC\\Core\\Migrations\\Version28000Date20230906104802' => __DIR__ . '/../../..' . '/core/Migrations/Version28000Date20230906104802.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', 'OC\\DB\\Adapter' => __DIR__ . '/../../..' . '/lib/private/DB/Adapter.php', @@ -1743,6 +1753,11 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\TextProcessing\\Manager' => __DIR__ . '/../../..' . '/lib/private/TextProcessing/Manager.php', 'OC\\TextProcessing\\RemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/TextProcessing/RemoveOldTasksBackgroundJob.php', 'OC\\TextProcessing\\TaskBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/TextProcessing/TaskBackgroundJob.php', + 'OC\\TextToImage\\Db\\Task' => __DIR__ . '/../../..' . '/lib/private/TextToImage/Db/Task.php', + 'OC\\TextToImage\\Db\\TaskMapper' => __DIR__ . '/../../..' . '/lib/private/TextToImage/Db/TaskMapper.php', + 'OC\\TextToImage\\Manager' => __DIR__ . '/../../..' . '/lib/private/TextToImage/Manager.php', + 'OC\\TextToImage\\RemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/TextToImage/RemoveOldTasksBackgroundJob.php', + 'OC\\TextToImage\\TaskBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/TextToImage/TaskBackgroundJob.php', 'OC\\Translation\\TranslationManager' => __DIR__ . '/../../..' . '/lib/private/Translation/TranslationManager.php', 'OC\\URLGenerator' => __DIR__ . '/../../..' . '/lib/private/URLGenerator.php', 'OC\\Updater' => __DIR__ . '/../../..' . '/lib/private/Updater.php', diff --git a/lib/private/Server.php b/lib/private/Server.php index 949a7ccfd3f4d..2b6b4fbe682da 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -1424,6 +1424,8 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(\OCP\TextProcessing\IManager::class, \OC\TextProcessing\Manager::class); + $this->registerAlias(\OCP\TextToImage\IManager::class, \OC\TextToImage\Manager::class); + $this->registerAlias(ILimiter::class, Limiter::class); $this->registerAlias(IPhoneNumberUtil::class, PhoneNumberUtil::class); From 92cc171a613059f0c15df9bcf26fdf54ef412683 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Thu, 19 Oct 2023 11:07:03 +0200 Subject: [PATCH 26/66] fix(TextToImage): Fix OpenAPI definitions Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 6 +- core/openapi.json | 807 +++++++++++++++++++ 2 files changed, 811 insertions(+), 2 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 921b3cbfb1178..402d298373283 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -60,9 +60,11 @@ public function __construct( } /** - * * Check whether this feature is available + * Check whether this feature is available * * @return DataResponse + * + * 200: Returns availability status */ #[PublicPage] public function isAvailable(): DataResponse { @@ -86,7 +88,7 @@ public function isAvailable(): DataResponse { #[PublicPage] #[UserRateLimit(limit: 20, period: 120)] #[AnonRateLimit(limit: 5, period: 120)] - public function schedule(string $input, string $type, string $appId, string $identifier = ''): DataResponse { + public function schedule(string $input, string $appId, string $identifier = ''): DataResponse { $task = new Task($input, $appId, $this->userId, $identifier); try { $this->textToImageManager->scheduleTask($task); diff --git a/core/openapi.json b/core/openapi.json index a9c810a790ae0..6f606ceb4cadf 100644 --- a/core/openapi.json +++ b/core/openapi.json @@ -450,6 +450,42 @@ } } }, + "TextToImageTask": { + "type": "object", + "required": [ + "id", + "status", + "userId", + "appId", + "input", + "identifier" + ], + "properties": { + "id": { + "type": "integer", + "format": "int64", + "nullable": true + }, + "status": { + "type": "integer", + "format": "int64" + }, + "userId": { + "type": "string", + "nullable": true + }, + "appId": { + "type": "string" + }, + "input": { + "type": "string" + }, + "identifier": { + "type": "string", + "nullable": true + } + } + }, "UnifiedSearchProvider": { "type": "object", "required": [ @@ -5130,6 +5166,777 @@ } } }, + "/ocs/v2.php/text2image/is_available": { + "get": { + "operationId": "text_to_image_api-is-available", + "summary": "Check whether this feature is available", + "tags": [ + "text_to_image_api" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Returns availability status", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "isAvailable" + ], + "properties": { + "isAvailable": { + "type": "boolean" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "/ocs/v2.php/text2image/schedule": { + "post": { + "operationId": "text_to_image_api-schedule", + "summary": "This endpoint allows scheduling a text to image task", + "tags": [ + "text_to_image_api" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "input", + "in": "query", + "description": "Input text", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "appId", + "in": "query", + "description": "ID of the app that will execute the task", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "identifier", + "in": "query", + "description": "An arbitrary identifier for the task", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Task scheduled successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "task" + ], + "properties": { + "task": { + "$ref": "#/components/schemas/TextToImageTask" + } + } + } + } + } + } + } + } + } + }, + "412": { + "description": "Scheduling task is not possible", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "/ocs/v2.php/text2image/task/{id}": { + "get": { + "operationId": "text_to_image_api-get-task", + "summary": "This endpoint allows checking the status and results of a task. Tasks are removed 1 week after receiving their last update.", + "tags": [ + "text_to_image_api" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The id of the task", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Task returned", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "task" + ], + "properties": { + "task": { + "$ref": "#/components/schemas/TextToImageTask" + } + } + } + } + } + } + } + } + } + }, + "404": { + "description": "Task not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + }, + "delete": { + "operationId": "text_to_image_api-delete-task", + "summary": "This endpoint allows to delete a scheduled task for a user", + "tags": [ + "text_to_image_api" + ], + "security": [ + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The id of the task", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Task returned", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "task" + ], + "properties": { + "task": { + "$ref": "#/components/schemas/TextToImageTask" + } + } + } + } + } + } + } + } + } + }, + "404": { + "description": "Task not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "/ocs/v2.php/text2image/task/{id}/image": { + "get": { + "operationId": "text_to_image_api-get-image", + "summary": "This endpoint allows downloading the resulting image of a task", + "tags": [ + "text_to_image_api" + ], + "security": [ + {}, + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The id of the task", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Image returned", + "content": { + "*/*": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "404": { + "description": "Task or image not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "/ocs/v2.php/text2image/tasks/app/{appId}": { + "get": { + "operationId": "text_to_image_api-list-tasks-by-app", + "summary": "This endpoint returns a list of tasks of a user that are related with a specific appId and optionally with an identifier", + "tags": [ + "text_to_image_api" + ], + "security": [ + { + "bearer_auth": [] + }, + { + "basic_auth": [] + } + ], + "parameters": [ + { + "name": "identifier", + "in": "query", + "description": "An arbitrary identifier for the task", + "schema": { + "type": "string", + "nullable": true + } + }, + { + "name": "appId", + "in": "path", + "description": "ID of the app", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "OCS-APIRequest", + "in": "header", + "description": "Required to be true for the API request to pass", + "required": true, + "schema": { + "type": "boolean", + "default": true + } + } + ], + "responses": { + "200": { + "description": "Task list returned", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "tasks" + ], + "properties": { + "tasks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TextToImageTask" + } + } + } + } + } + } + } + } + } + } + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + }, "/status.php": { "get": { "operationId": "get-status", From 8968573d9fa0b9abac1dd5c7684dd94258a080cf Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 12:17:17 +0200 Subject: [PATCH 27/66] enh(TextToImage): Add getExpectedRuntime to IProvider and run tasks during request lifetime if possible Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 9 +++++- lib/private/TextToImage/Manager.php | 32 +++++++++++++++++++- lib/public/TextToImage/IManager.php | 11 ++++++- lib/public/TextToImage/IProvider.php | 6 ++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index 402d298373283..c7878d7cdc3b8 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -36,6 +36,7 @@ use OCP\AppFramework\Http\Attribute\UserRateLimit; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\FileDisplayResponse; +use OCP\DB\Exception; use OCP\Files\NotFoundException; use OCP\IL10N; use OCP\IRequest; @@ -91,7 +92,11 @@ public function isAvailable(): DataResponse { public function schedule(string $input, string $appId, string $identifier = ''): DataResponse { $task = new Task($input, $appId, $this->userId, $identifier); try { - $this->textToImageManager->scheduleTask($task); + try { + $this->textToImageManager->runOrScheduleTask($task); + } catch (\RuntimeException) { + // noop + } $json = $task->jsonSerialize(); @@ -100,6 +105,8 @@ public function schedule(string $input, string $appId, string $identifier = ''): ]); } catch (PreConditionNotMetException) { return new DataResponse(['message' => $this->l->t('No text to image provider is available')], Http::STATUS_PRECONDITION_FAILED); + } catch (Exception) { + return new DataResponse(['message' => $this->l->t('Internal error')], Http::STATUS_INTERNAL_SERVER_ERROR); } } diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index 181748caf9979..c309b7264e197 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -187,12 +187,12 @@ public function runTask(Task $task): void { /** * @inheritDoc - * @throws Exception */ public function scheduleTask(Task $task): void { if (!$this->hasProviders()) { throw new PreConditionNotMetException('No text to image provider is installed that can handle this task'); } + $this->logger->warning('Scheduling Text2Image Task'); $task->setStatus(Task::STATUS_SCHEDULED); $taskEntity = DbTask::fromPublicTask($task); $this->taskMapper->insert($taskEntity); @@ -202,6 +202,36 @@ public function scheduleTask(Task $task): void { ]); } + /** + * @inheritDoc + */ + public function runOrScheduleTask(Task $task) : void { + if (!$this->hasProviders()) { + throw new PreConditionNotMetException('No text to image provider is installed that can handle this task'); + } + $providers = $this->getProviders(); + + $json = $this->config->getAppValue('core', 'ai.text2image_provider', ''); + if ($json !== '') { + try { + $className = json_decode($json, true, 512, JSON_THROW_ON_ERROR); + $provider = current(array_filter($providers, fn ($provider) => $provider::class === $className)); + if ($provider !== false) { + $providers = [$provider]; + } + } catch (\JsonException $e) { + $this->logger->warning('Failed to decode Text2Image setting `ai.text2image_provider`', ['exception' => $e]); + } + } + $maxExecutionTime = (int) ini_get('max_execution_time'); + // Offload the tttttttask to a background job if the expected runtime of the likely provider is longer than 80% of our max execution time + if ($providers[0]->getExpectedRuntime() > $maxExecutionTime * 0.8) { + $this->scheduleTask($task); + return; + } + $this->runTask($task); + } + /** * @inheritDoc */ diff --git a/lib/public/TextToImage/IManager.php b/lib/public/TextToImage/IManager.php index c5c8a0a22a60e..cd97312779ce1 100644 --- a/lib/public/TextToImage/IManager.php +++ b/lib/public/TextToImage/IManager.php @@ -26,6 +26,7 @@ namespace OCP\TextToImage; +use OCP\DB\Exception; use OCP\PreConditionNotMetException; use OCP\TextToImage\Exception\TaskNotFoundException; use RuntimeException; @@ -55,11 +56,19 @@ public function runTask(Task $task): void; * If inference fails a \OCP\TextToImage\Events\TaskFailedEvent will be dispatched instead * * @param Task $task The task to schedule - * @throws PreConditionNotMetException If no or not the requested provider was registered but this method was still called + * @throws PreConditionNotMetException If no provider was registered but this method was still called + * @throws Exception If there was a problem inserting the task into the database * @since 28.0.0 */ public function scheduleTask(Task $task) : void; + /** + * @throws Exception if there was a problem inserting the task into the database + * @throws PreConditionNotMetException if no provider is registered + * @throws RuntimeException If the task run fail + */ + public function runOrScheduleTask(Task $task) : void; + /** * Delete a task that has been scheduled before * diff --git a/lib/public/TextToImage/IProvider.php b/lib/public/TextToImage/IProvider.php index 4fc7308924308..12cf39bb713a7 100644 --- a/lib/public/TextToImage/IProvider.php +++ b/lib/public/TextToImage/IProvider.php @@ -49,4 +49,10 @@ public function getName(): string; * @throws RuntimeException If the text could not be processed */ public function generate(string $prompt, $resource): void; + + /** + * The expected runtime for one task with this provider in seconds + * @since 28.0.0 + */ + public function getExpectedRuntime(): int; } From b7fd5185b69be28338110cc1dc1655883d84c302 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:13:15 +0200 Subject: [PATCH 28/66] enh(TextToImage): Allow generating multiple images with one task Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 14 ++++++--- core/ResponseDefinitions.php | 1 + core/routes.php | 2 +- lib/private/TextToImage/Db/Task.php | 26 +++++---------- lib/private/TextToImage/Manager.php | 28 +++++++++++------ lib/public/TextToImage/IProvider.php | 4 +-- lib/public/TextToImage/Task.php | 33 ++++++++++++++------ 7 files changed, 64 insertions(+), 44 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index c7878d7cdc3b8..aee3a462f5f34 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -80,6 +80,7 @@ public function isAvailable(): DataResponse { * @param string $input Input text * @param string $appId ID of the app that will execute the task * @param string $identifier An arbitrary identifier for the task + * @param int $numberOfImages The number of images to generate * * @return DataResponse|DataResponse * @@ -89,8 +90,8 @@ public function isAvailable(): DataResponse { #[PublicPage] #[UserRateLimit(limit: 20, period: 120)] #[AnonRateLimit(limit: 5, period: 120)] - public function schedule(string $input, string $appId, string $identifier = ''): DataResponse { - $task = new Task($input, $appId, $this->userId, $identifier); + public function schedule(string $input, string $appId, string $identifier = '', int $numberOfImages = 8): DataResponse { + $task = new Task($input, $appId, $numberOfImages, $this->userId, $identifier); try { try { $this->textToImageManager->runOrScheduleTask($task); @@ -145,6 +146,7 @@ public function getTask(int $id): DataResponse { * This endpoint allows downloading the resulting image of a task * * @param int $id The id of the task + * @param int $index The index of the image to retrieve * * @return FileDisplayResponse|DataResponse * @@ -153,15 +155,17 @@ public function getTask(int $id): DataResponse { */ #[PublicPage] #[BruteForceProtection(action: 'text2image')] - public function getImage(int $id): DataResponse|FileDisplayResponse { + public function getImage(int $id, int $index): DataResponse|FileDisplayResponse { try { $task = $this->textToImageManager->getUserTask($id, $this->userId); try { $folder = $this->appData->getFolder('text2image'); } catch(NotFoundException) { - $folder = $this->appData->newFolder('text2image'); + $res = new DataResponse(['message' => $this->l->t('Image not found')], Http::STATUS_NOT_FOUND); + $res->throttle(['action' => 'text2image']); + return $res; } - $file = $folder->getFile((string)$task->getId()); + $file = $folder->getFolder((string) $task->getId())->getFile((string) $index); $info = getimagesizefromstring($file->getContent()); return new FileDisplayResponse($file, Http::STATUS_OK, ['Content-Type' => image_type_to_mime_type($info[2])]); diff --git a/core/ResponseDefinitions.php b/core/ResponseDefinitions.php index 2548880395aec..103d4f84a7f28 100644 --- a/core/ResponseDefinitions.php +++ b/core/ResponseDefinitions.php @@ -152,6 +152,7 @@ * appId: string, * input: string, * identifier: ?string, + * numberOfImages: int * } */ class ResponseDefinitions { diff --git a/core/routes.php b/core/routes.php index a779b130ba25f..fe1fe6fcd7500 100644 --- a/core/routes.php +++ b/core/routes.php @@ -159,7 +159,7 @@ ['root' => '/text2image', 'name' => 'TextToImageApi#isAvailable', 'url' => '/is_available', 'verb' => 'GET'], ['root' => '/text2image', 'name' => 'TextToImageApi#schedule', 'url' => '/schedule', 'verb' => 'POST'], ['root' => '/text2image', 'name' => 'TextToImageApi#getTask', 'url' => '/task/{id}', 'verb' => 'GET'], - ['root' => '/text2image', 'name' => 'TextToImageApi#getImage', 'url' => '/task/{id}/image', 'verb' => 'GET'], + ['root' => '/text2image', 'name' => 'TextToImageApi#getImage', 'url' => '/task/{id}/image/{index}', 'verb' => 'GET'], ['root' => '/text2image', 'name' => 'TextToImageApi#deleteTask', 'url' => '/task/{id}', 'verb' => 'DELETE'], ['root' => '/text2image', 'name' => 'TextToImageApi#listTasksByApp', 'url' => '/tasks/app/{appId}', 'verb' => 'GET'], ], diff --git a/lib/private/TextToImage/Db/Task.php b/lib/private/TextToImage/Db/Task.php index bff827533a59d..12818d956c203 100644 --- a/lib/private/TextToImage/Db/Task.php +++ b/lib/private/TextToImage/Db/Task.php @@ -27,10 +27,6 @@ use DateTime; use OCP\AppFramework\Db\Entity; -use OCP\Files\AppData\IAppDataFactory; -use OCP\Files\NotFoundException; -use OCP\Files\NotPermittedException; -use OCP\Image; use OCP\TextToImage\Task as OCPTask; /** @@ -48,6 +44,8 @@ * @method string getAppId() * @method setIdentifier(string $identifier) * @method string|null getIdentifier() + * @method setNumberOfImages(int $numberOfImages) + * @method int getNumberOfImages() */ class Task extends Entity { protected $lastUpdated; @@ -57,16 +55,17 @@ class Task extends Entity { protected $userId; protected $appId; protected $identifier; + protected $numberOfImages; /** * @var string[] */ - public static array $columns = ['id', 'last_updated', 'input', 'status', 'user_id', 'app_id', 'identifier']; + public static array $columns = ['id', 'last_updated', 'input', 'status', 'user_id', 'app_id', 'identifier', 'number_of_images']; /** * @var string[] */ - public static array $fields = ['id', 'lastUpdated', 'input', 'status', 'userId', 'appId', 'identifier']; + public static array $fields = ['id', 'lastUpdated', 'input', 'status', 'userId', 'appId', 'identifier', 'numberOfImages']; public function __construct() { @@ -78,6 +77,7 @@ public function __construct() { $this->addType('userId', 'string'); $this->addType('appId', 'string'); $this->addType('identifier', 'string'); + $this->addType('numberOfImages', 'integer'); } public function toRow(): array { @@ -92,6 +92,7 @@ public static function fromPublicTask(OCPTask $task): Task { 'id' => $task->getId(), 'lastUpdated' => time(), 'status' => $task->getStatus(), + 'numberOfImages' => $task->getNumberOfImages(), 'input' => $task->getInput(), 'userId' => $task->getUserId(), 'appId' => $task->getAppId(), @@ -101,20 +102,9 @@ public static function fromPublicTask(OCPTask $task): Task { } public function toPublicTask(): OCPTask { - $task = new OCPTask($this->getInput(), $this->getAppId(), $this->getuserId(), $this->getIdentifier()); + $task = new OCPTask($this->getInput(), $this->getAppId(), $this->getNumberOfImages(), $this->getuserId(), $this->getIdentifier()); $task->setId($this->getId()); $task->setStatus($this->getStatus()); - $appData = \OC::$server->get(IAppDataFactory::class)->get('core'); - try { - try { - $folder = $appData->getFolder('text2image'); - } catch(NotFoundException) { - $folder = $appData->newFolder('text2image'); - } - $task->setOutputImage(new Image(base64_encode($folder->getFile((string)$task->getId())->getContent()))); - } catch (NotFoundException|NotPermittedException) { - // noop - } return $task; } } diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index c309b7264e197..a48b202239d0a 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -139,17 +139,27 @@ public function runTask(Task $task): void { $this->logger->debug('Creating folder in appdata for Text2Image results'); $folder = $this->appData->newFolder('text2image'); } - $this->logger->debug('Creating result file for Text2Image task'); - $file = $folder->newFile((string) $task->getId()); - $resource = $file->write(); - if ($resource === false) { - throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: Couldn\'t open file to write.'); + try { + $folder = $folder->getFolder((string) $task->getId()); + } catch(NotFoundException) { + $this->logger->debug('Creating new folder in appdata Text2Image results folder'); + $folder = $this->appData->newFolder((string) $task->getId()); + } + $this->logger->debug('Creating result files for Text2Image task'); + $resources = []; + for ($i = 0; $i < $task->getNumberOfImages(); $i++) { + $resources[] = $folder->newFile((string) $i)->write(); + if ($resource[count($resources) - 1] === false) { + throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: Couldn\'t open file to write.'); + } } $this->logger->debug('Calling Text2Image provider\'s generate method'); - $provider->generate($task->getInput(), $resource); - if (is_resource($resource)) { - // If $resource hasn't been closed yet, we'll do that here - fclose($resource); + $provider->generate($task->getInput(), $resources); + for ($i = 0; $i < $task->getNumberOfImages(); $i++) { + if (is_resource($resources[$i])) { + // If $resource hasn't been closed yet, we'll do that here + fclose($resource[$i]); + } } $task->setStatus(Task::STATUS_SUCCESSFUL); $this->logger->debug('Updating Text2Image task in DB'); diff --git a/lib/public/TextToImage/IProvider.php b/lib/public/TextToImage/IProvider.php index 12cf39bb713a7..789a69ade679c 100644 --- a/lib/public/TextToImage/IProvider.php +++ b/lib/public/TextToImage/IProvider.php @@ -43,12 +43,12 @@ public function getName(): string; * Processes a text * * @param string $prompt The input text - * @param resource $resource The file resource to write the image to + * @param resource[] $resources The file resources to write the images to * @return void * @since 28.0.0 * @throws RuntimeException If the text could not be processed */ - public function generate(string $prompt, $resource): void; + public function generate(string $prompt, array $resources): void; /** * The expected runtime for one task with this provider in seconds diff --git a/lib/public/TextToImage/Task.php b/lib/public/TextToImage/Task.php index 545bd8bac5a02..2f9869bc55ac1 100644 --- a/lib/public/TextToImage/Task.php +++ b/lib/public/TextToImage/Task.php @@ -25,7 +25,11 @@ namespace OCP\TextToImage; +use OCP\Files\AppData\IAppDataFactory; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; use OCP\IImage; +use OCP\Image; /** * This is a text to image task @@ -35,8 +39,6 @@ final class Task implements \JsonSerializable { protected ?int $id = null; - private ?IImage $image = null; - /** * @since 28.0.0 */ @@ -66,6 +68,7 @@ final class Task implements \JsonSerializable { /** * @param string $input * @param string $appId + * @param int $numberOfImages * @param string|null $userId * @param null|string $identifier An arbitrary identifier for this task. max length: 255 chars * @since 28.0.0 @@ -73,25 +76,36 @@ final class Task implements \JsonSerializable { final public function __construct( protected string $input, protected string $appId, + protected int $numberOfImages, protected ?string $userId, protected ?string $identifier = '', ) { } /** - * @return IImage|null + * @return IImage[]|null * @since 28.0.0 */ - final public function getOutputImage(): ?IImage { - return $this->image; + final public function getOutputImages(): ?array { + $appData = \OC::$server->get(IAppDataFactory::class)->get('core'); + try { + $folder = $appData->getFolder('text2image')->getFolder((string)$this->getId()); + $images = []; + for ($i = 0; $i < $this->getNumberOfImages(); $i++) { + $images[] = new Image(base64_encode($folder->getFile((string) $i)->getContent())); + } + return $images; + } catch (NotFoundException|NotPermittedException) { + return null; + } } /** - * @param IImage|null $image + * @return int * @since 28.0.0 */ - final public function setOutputImage(?IImage $image): void { - $this->image = $image; + final public function getNumberOfImages(): int { + return $this->numberOfImages; } /** @@ -159,7 +173,7 @@ final public function getUserId(): ?string { } /** - * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, identifier: ?string} + * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, identifier: ?string, numberOfImages: int} * @since 28.0.0 */ public function jsonSerialize(): array { @@ -168,6 +182,7 @@ public function jsonSerialize(): array { 'status' => $this->getStatus(), 'userId' => $this->getUserId(), 'appId' => $this->getAppId(), + 'numberOfImages' => $this->getNumberOfImages(), 'input' => $this->getInput(), 'identifier' => $this->getIdentifier(), ]; From 8ab47b64b5fd4ec921d167e418768eae108a9041 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:13:48 +0200 Subject: [PATCH 29/66] enh(testing app): Add fake text2image provider Signed-off-by: Marcel Klehr --- apps/testing/img/logo.png | Bin 0 -> 3571 bytes apps/testing/lib/AppInfo/Application.php | 2 + .../lib/Provider/FakeText2ImageProvider.php | 43 ++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 apps/testing/img/logo.png create mode 100644 apps/testing/lib/Provider/FakeText2ImageProvider.php diff --git a/apps/testing/img/logo.png b/apps/testing/img/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..df32e1c7eab4441c0bde532eb93e95c01d96f479 GIT binary patch literal 3571 zcmeHJ=QkUQ1C3405y`Y1B>> zEnck{wfC?8$M4;9?!D)Jz2|_7+*006L?niwGe_}L#uGSmIhAE=;W0DyMS z#sX#dCtm)C{~GvzYQVHFMDow>Ps5N_wg4JhI(i01CT12OD;qlpC)X8j9uO~>k6!>H zC?qT*Dkd%=c~weUMiwe3e@#L0y3$|DDynMg8d};qH*e|c>BHa#hDOFFre@|AmR8m_ z2&65_&fej+qm#3XtDC#W9ZxTBAK$xZKmWh)1q22Khu#l+5FQa375y+KHttb;0wxiQ zOGJF$1L1k&%U7@8ylrZ3dDq(3 z{{BP9$IhvFJMEM@YjDjay4G_#qw z+2v=l7l#b%>emYSL=VZ-YO+KBCFaDdtSBUpS8Cw>fx7gmD|O0#lf+79#vd6=B>a4l!~*+_Y3lDX z0yZAJ&!cvgT@N4_THu7A=YYopV~LioDKEG(tMbPf?;48%TFBb89Rr<;EC?bWqXZ zv6q)1TtD0E%%P0~2+4@u*o`XA=B%Z<=w6PU8foC{T4ds$8i^)%b-ZVMGBK_1v$~2y z*{Ra^6Wq_2@0^&NdU^&FfYqfd<6%)dlBG)93g_}J7v!|WbqAcoZB(ngsI{imWqLns zBm@y7f&+3JCHFibMu=1;d~7w+^OZZ-I$PV8<9!>^b|04<_vHah;;iK6hb+cD0ZaoE`+E4sf+Fb|qi&2-%DlSRKD*v=%` zZ;y(lhTdR;>u=D6|gbk@yK+3{32@2pw%a zEa5uM1kWZSC814SS{5wF17wA0|nOQt{~90=OwU8O|b;01BV{>vWi-kBP^^- zulF|4$AGP>))+zPcl0NVgkvNOhbz3qaYoncYE>q7>XQ8iV}l3M69Xa9`7n3*5Eqb& z9CWsTb5_w4N}kR3GesMA%U4a&6{GD#Zmc+3HmT3fH(d?dV$!hPH1Nr3%R{awRXh=2 zM~|?f-(&x66u>GcNh8I!r+y1l*2tVUEJ$ zAZBDn@~@CXl+#Q{ca#I7W9H%LIjk4k})-K1z>31e_Pt&GPC$W9d}47UA*;McZHKmjOF*^yu*SvdffR+39#1yxBH_ogUF>A>I$~B^@jLnr~Qgn-3>>n`_CQWIiuSu8kWbvBE~V$lnlF#QT6rNoOWKRs!(X|;l!Z7W*vF()h-$t zcXQ=9O?o*tD#WhDHUQ;vt0vT}c(7RYCwYJ1aQ^bmkfCG~bWg51Y|my1lm;pt`am(1 z3|g>e2iC884Bx|h8^Q)6Kt=CUi+-`*%%GM6~@E4-Z2 zJX~vIqz1Td(+gxG`i1A^QD4x$5&_-k#7gD1A~qBvaY0o+tCl~!Fv;NM@II||+3>bo zjYnCYW%g4064~IW{z}Xgdek7N0G9Hi#ba(Pry!BrE7fmU{+r+?~q}4BJJv?X81>SJxTv1`&bfqZ?rH`}x6 zsZyTXr**3X2gFSDwi^dKD2^=Ts~Qu=DHX2E&(fZ`yf@Ue`2BONe|}yhWhX=7OSOAD zRovmU)Ad_NhAP%C*3B$z%y=h>??dnH>P6Ij?Wo?j4C@i1VTfTA|N6<#B+rvzna*)0 z@Px>z$?W^=_xp7w~ zi_^Xvm5ol`RHd`JS!2oz!V|q6wt#m~V8Y@jz|jHV_=No_^Rz#u0=o$$iP=(5T0~AFHkR|4&E90+1(8gCQ+nZ7 zItUpt{#M!yc=ZmAb||L2dSZ~`b{a)}nX?yTeKNX~wyPVrLYT`9nTz^$E_gregisterTranslationProvider(FakeTranslationProvider::class); $context->registerTextProcessingProvider(FakeTextProcessingProvider::class); + $context->registerTextToImageProvider(FakeText2ImageProvider::class); } public function boot(IBootContext $context): void { diff --git a/apps/testing/lib/Provider/FakeText2ImageProvider.php b/apps/testing/lib/Provider/FakeText2ImageProvider.php new file mode 100644 index 0000000000000..20efe427e1639 --- /dev/null +++ b/apps/testing/lib/Provider/FakeText2ImageProvider.php @@ -0,0 +1,43 @@ + + * + * @author Marcel Klehr + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +namespace OCA\Testing\Provider; + +use OCP\TextToImage\IProvider; + +class FakeText2ImageProvider implements IProvider { + + public function getName(): string { + return 'Fake Text2Image provider'; + } + + public function generate(string $prompt, array $resources): void { + foreach ($resources as $resource) { + $read = fopen(__DIR__ . '/../../img/logo.png', 'r'); + stream_copy_to_stream($read, $resource); + } + } + + public function getExpectedRuntime(): int { + return 1; + } +} From 37c4ccc751ad0f1bfbd6c85e33a8c16bdd79ca5c Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:15:40 +0200 Subject: [PATCH 30/66] fix(Text2Image): Fix Task#lastUpdated initialization to use DateTime instead of time() Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Db/Task.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/TextToImage/Db/Task.php b/lib/private/TextToImage/Db/Task.php index 12818d956c203..8d857ce47eac8 100644 --- a/lib/private/TextToImage/Db/Task.php +++ b/lib/private/TextToImage/Db/Task.php @@ -90,7 +90,7 @@ public static function fromPublicTask(OCPTask $task): Task { /** @var Task $dbTask */ $dbTask = Task::fromParams([ 'id' => $task->getId(), - 'lastUpdated' => time(), + 'lastUpdated' => new DateTime('now'), 'status' => $task->getStatus(), 'numberOfImages' => $task->getNumberOfImages(), 'input' => $task->getInput(), From 7b7f552a6e70223c7eec7300cba2eac5d96bb765 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:33:05 +0200 Subject: [PATCH 31/66] fix(Text2Image): Fix OpenAPI types Signed-off-by: Marcel Klehr --- core/Controller/TextToImageApiController.php | 2 +- core/openapi.json | 67 +++++++++++++++++++- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/core/Controller/TextToImageApiController.php b/core/Controller/TextToImageApiController.php index aee3a462f5f34..27833c0035dc6 100644 --- a/core/Controller/TextToImageApiController.php +++ b/core/Controller/TextToImageApiController.php @@ -82,7 +82,7 @@ public function isAvailable(): DataResponse { * @param string $identifier An arbitrary identifier for the task * @param int $numberOfImages The number of images to generate * - * @return DataResponse|DataResponse + * @return DataResponse|DataResponse * * 200: Task scheduled successfully * 412: Scheduling task is not possible diff --git a/core/openapi.json b/core/openapi.json index 6f606ceb4cadf..2bf52d0895d82 100644 --- a/core/openapi.json +++ b/core/openapi.json @@ -458,7 +458,8 @@ "userId", "appId", "input", - "identifier" + "identifier", + "numberOfImages" ], "properties": { "id": { @@ -483,6 +484,10 @@ "identifier": { "type": "string", "nullable": true + }, + "numberOfImages": { + "type": "integer", + "format": "int64" } } }, @@ -5280,6 +5285,16 @@ "default": "" } }, + { + "name": "numberOfImages", + "in": "query", + "description": "The number of images to generate", + "schema": { + "type": "integer", + "format": "int64", + "default": 8 + } + }, { "name": "OCS-APIRequest", "in": "header", @@ -5367,6 +5382,44 @@ } } } + }, + "500": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ocs" + ], + "properties": { + "ocs": { + "type": "object", + "required": [ + "meta", + "data" + ], + "properties": { + "meta": { + "$ref": "#/components/schemas/OCSMeta" + }, + "data": { + "type": "object", + "required": [ + "message" + ], + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } } } } @@ -5680,7 +5733,7 @@ } } }, - "/ocs/v2.php/text2image/task/{id}/image": { + "/ocs/v2.php/text2image/task/{id}/image/{index}": { "get": { "operationId": "text_to_image_api-get-image", "summary": "This endpoint allows downloading the resulting image of a task", @@ -5707,6 +5760,16 @@ "format": "int64" } }, + { + "name": "index", + "in": "path", + "description": "The index of the image to retrieve", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, { "name": "OCS-APIRequest", "in": "header", From bc85acf7d0f7b66211a011e9a192a5e73ddf27de Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:38:20 +0200 Subject: [PATCH 32/66] fix(Text2Image): Fix psalm issues Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Manager.php | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index a48b202239d0a..f7abc791a2d44 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -147,8 +147,11 @@ public function runTask(Task $task): void { } $this->logger->debug('Creating result files for Text2Image task'); $resources = []; + $files = []; for ($i = 0; $i < $task->getNumberOfImages(); $i++) { - $resources[] = $folder->newFile((string) $i)->write(); + $file = $folder->newFile((string) $i); + $files[] = $file; + $resources[] = $file->write(); if ($resource[count($resources) - 1] === false) { throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: Couldn\'t open file to write.'); } @@ -158,7 +161,7 @@ public function runTask(Task $task): void { for ($i = 0; $i < $task->getNumberOfImages(); $i++) { if (is_resource($resources[$i])) { // If $resource hasn't been closed yet, we'll do that here - fclose($resource[$i]); + fclose($resources[$i]); } } $task->setStatus(Task::STATUS_SUCCESSFUL); @@ -166,17 +169,20 @@ public function runTask(Task $task): void { $this->taskMapper->update(DbTask::fromPublicTask($task)); return; } catch (\RuntimeException|\Throwable $e) { - if (isset($resource) && is_resource($resource)) { - // If $resource hasn't been closed yet, we'll do that here - fclose($resource); - } - try { - if (isset($file)) { - $file->delete(); + for ($i = 0; $i < $task->getNumberOfImages(); $i++) { + if (isset($resources[$i]) && is_resource($resources[$i])) { + // If $resource hasn't been closed yet, we'll do that here + fclose($resources[$i]); + } + if (isset($files, $files[$i])) { + try { + $files[$i]->delete(); + } catch(NotPermittedException $e) { + $this->logger->warning('Failed to clean up Text2Image result file after error', ['exception' => $e]); + } } - } catch(NotPermittedException $e) { - $this->logger->warning('Failed to clean up Text2Image result file after error', ['exception' => $e]); } + $this->logger->info('Text2Image generation using provider ' . $provider->getName() . ' failed', ['exception' => $e]); $task->setStatus(Task::STATUS_FAILED); try { From 73da7f20d32cdbb3e14dbeae6cc43a8573f4ebf9 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:53:06 +0200 Subject: [PATCH 33/66] fix(Text2Image): Add number_of_images to migration Signed-off-by: Marcel Klehr --- core/Migrations/Version28000Date20230906104802.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/Migrations/Version28000Date20230906104802.php b/core/Migrations/Version28000Date20230906104802.php index 7134899c82e6b..5a51063922902 100644 --- a/core/Migrations/Version28000Date20230906104802.php +++ b/core/Migrations/Version28000Date20230906104802.php @@ -61,6 +61,10 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt 'length' => 6, 'default' => 0, ]); + $table->addColumn('number_of_images', Types::INTEGER, [ + 'notnull' => true, + 'default' => 1, + ]); $table->addColumn('user_id', Types::STRING, [ 'notnull' => false, 'length' => 64, From 4c58edc1b7fcc042c1cff1f1123732d8526565be Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:53:32 +0200 Subject: [PATCH 34/66] fix(Text2Image): Fix psalm error Signed-off-by: Marcel Klehr --- lib/private/TextToImage/Manager.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/TextToImage/Manager.php b/lib/private/TextToImage/Manager.php index f7abc791a2d44..0b61d1af2465d 100644 --- a/lib/private/TextToImage/Manager.php +++ b/lib/private/TextToImage/Manager.php @@ -152,7 +152,7 @@ public function runTask(Task $task): void { $file = $folder->newFile((string) $i); $files[] = $file; $resources[] = $file->write(); - if ($resource[count($resources) - 1] === false) { + if ($resources[count($resources) - 1] === false) { throw new RuntimeException('Text2Image generation using provider ' . $provider->getName() . ' failed: Couldn\'t open file to write.'); } } From 9ee72633cf78a777756d2d4afe707f2539521ccc Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 13:54:36 +0200 Subject: [PATCH 35/66] enh(Text2Image): Expose expected completion time Signed-off-by: Marcel Klehr --- .../Version28000Date20230906104802.php | 3 +++ core/ResponseDefinitions.php | 3 ++- core/openapi.json | 7 +++++- lib/private/TextToImage/Db/Task.php | 10 +++++++-- lib/public/TextToImage/Task.php | 22 ++++++++++++++++++- 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/core/Migrations/Version28000Date20230906104802.php b/core/Migrations/Version28000Date20230906104802.php index 5a51063922902..3d8ee9a66e087 100644 --- a/core/Migrations/Version28000Date20230906104802.php +++ b/core/Migrations/Version28000Date20230906104802.php @@ -82,6 +82,9 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt $table->addColumn('last_updated', Types::DATETIME, [ 'notnull' => false, ]); + $table->addColumn('completion_expeted_at', Types::DATETIME, [ + 'notnull' => false, + ]); $table->setPrimaryKey(['id'], 't2i_tasks_id_index'); $table->addIndex(['last_updated'], 't2i_tasks_updated'); diff --git a/core/ResponseDefinitions.php b/core/ResponseDefinitions.php index 103d4f84a7f28..82d3d6e05ae1c 100644 --- a/core/ResponseDefinitions.php +++ b/core/ResponseDefinitions.php @@ -152,7 +152,8 @@ * appId: string, * input: string, * identifier: ?string, - * numberOfImages: int + * numberOfImages: int, + * completionExpectedAt: int, * } */ class ResponseDefinitions { diff --git a/core/openapi.json b/core/openapi.json index 2bf52d0895d82..5fed9716c5cfe 100644 --- a/core/openapi.json +++ b/core/openapi.json @@ -459,7 +459,8 @@ "appId", "input", "identifier", - "numberOfImages" + "numberOfImages", + "completionExpectedAt" ], "properties": { "id": { @@ -488,6 +489,10 @@ "numberOfImages": { "type": "integer", "format": "int64" + }, + "completionExpectedAt": { + "type": "integer", + "format": "int64" } } }, diff --git a/lib/private/TextToImage/Db/Task.php b/lib/private/TextToImage/Db/Task.php index 8d857ce47eac8..312ed07880b10 100644 --- a/lib/private/TextToImage/Db/Task.php +++ b/lib/private/TextToImage/Db/Task.php @@ -46,6 +46,8 @@ * @method string|null getIdentifier() * @method setNumberOfImages(int $numberOfImages) * @method int getNumberOfImages() + * @method setCompletionExpectedAt(DateTime $at) + * @method DateTime getCompletionExpectedAt() */ class Task extends Entity { protected $lastUpdated; @@ -56,16 +58,17 @@ class Task extends Entity { protected $appId; protected $identifier; protected $numberOfImages; + protected $completionExpectedAt; /** * @var string[] */ - public static array $columns = ['id', 'last_updated', 'input', 'status', 'user_id', 'app_id', 'identifier', 'number_of_images']; + public static array $columns = ['id', 'last_updated', 'input', 'status', 'user_id', 'app_id', 'identifier', 'number_of_images', 'completion_expected_at']; /** * @var string[] */ - public static array $fields = ['id', 'lastUpdated', 'input', 'status', 'userId', 'appId', 'identifier', 'numberOfImages']; + public static array $fields = ['id', 'lastUpdated', 'input', 'status', 'userId', 'appId', 'identifier', 'numberOfImages', 'completionExpectedAt']; public function __construct() { @@ -78,6 +81,7 @@ public function __construct() { $this->addType('appId', 'string'); $this->addType('identifier', 'string'); $this->addType('numberOfImages', 'integer'); + $this->addType('completionExpectedAt', 'datetime'); } public function toRow(): array { @@ -97,6 +101,7 @@ public static function fromPublicTask(OCPTask $task): Task { 'userId' => $task->getUserId(), 'appId' => $task->getAppId(), 'identifier' => $task->getIdentifier(), + 'completionExpectedAt' => $task->getCompletionExpectedAt(), ]); return $dbTask; } @@ -105,6 +110,7 @@ public function toPublicTask(): OCPTask { $task = new OCPTask($this->getInput(), $this->getAppId(), $this->getNumberOfImages(), $this->getuserId(), $this->getIdentifier()); $task->setId($this->getId()); $task->setStatus($this->getStatus()); + $task->setCompletionExpectedAt($this->getCompletionExpectedAt()); return $task; } } diff --git a/lib/public/TextToImage/Task.php b/lib/public/TextToImage/Task.php index 2f9869bc55ac1..20ae32e0f6bc4 100644 --- a/lib/public/TextToImage/Task.php +++ b/lib/public/TextToImage/Task.php @@ -25,6 +25,7 @@ namespace OCP\TextToImage; +use DateTime; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; @@ -39,6 +40,8 @@ final class Task implements \JsonSerializable { protected ?int $id = null; + protected DateTime $completionExpectedAt; + /** * @since 28.0.0 */ @@ -124,6 +127,22 @@ final public function setStatus(int $status): void { $this->status = $status; } + /** + * @param DateTime $at + * @since 28.0.0 + */ + final public function setCompletionExpectedAt(DateTime $at): void { + $this->completionExpectedAt = $at; + } + + /** + * @return DateTime + * @since 28.0.0 + */ + final public function getCompletionExpectedAt(): DateTime { + return $this->completionExpectedAt; + } + /** * @return int|null * @since 28.0.0 @@ -173,7 +192,7 @@ final public function getUserId(): ?string { } /** - * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, identifier: ?string, numberOfImages: int} + * @psalm-return array{id: ?int, status: 0|1|2|3|4, userId: ?string, appId: string, input: string, identifier: ?string, numberOfImages: int, completionExpectedAt: int} * @since 28.0.0 */ public function jsonSerialize(): array { @@ -185,6 +204,7 @@ public function jsonSerialize(): array { 'numberOfImages' => $this->getNumberOfImages(), 'input' => $this->getInput(), 'identifier' => $this->getIdentifier(), + 'completionExpectedAt' => $this->getCompletionExpectedAt()->getTimestamp(), ]; } } From cee5aa84f02de18084067a2e33e8d252f2362be4 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Fri, 20 Oct 2023 14:56:24 +0200 Subject: [PATCH 36/66] fix(Text2Image): Fix psalm errors Signed-off-by: Marcel Klehr --- .../AppFramework/Bootstrap/IRegistrationContext.php | 11 +++++++++++ lib/public/TextToImage/IManager.php | 1 + 2 files changed, 12 insertions(+) diff --git a/lib/public/AppFramework/Bootstrap/IRegistrationContext.php b/lib/public/AppFramework/Bootstrap/IRegistrationContext.php index c34cec38eb127..b956c308c010c 100644 --- a/lib/public/AppFramework/Bootstrap/IRegistrationContext.php +++ b/lib/public/AppFramework/Bootstrap/IRegistrationContext.php @@ -38,6 +38,7 @@ use OCP\Files\Template\ICustomTemplateProvider; use OCP\IContainer; use OCP\TextProcessing\IProvider as ITextProcessingProvider; +use OCP\TextToImage\IProvider as ITextToImageProvider; use OCP\Notification\INotifier; use OCP\Preview\IProviderV2; use OCP\SpeechToText\ISpeechToTextProvider; @@ -230,6 +231,16 @@ public function registerSpeechToTextProvider(string $providerClass): void; */ public function registerTextProcessingProvider(string $providerClass): void; + /** + * Register a custom text2image provider class that provides the possibility to generate images + * through the OCP\TextToImage APIs + * + * @param string $providerClass + * @psalm-param class-string $providerClass + * @since 27.1.0 + */ + public function registerTextToImageProvider(string $providerClass): void; + /** * Register a custom template provider class that is able to inject custom templates * in addition to the user defined ones diff --git a/lib/public/TextToImage/IManager.php b/lib/public/TextToImage/IManager.php index cd97312779ce1..5afaecd37bc0c 100644 --- a/lib/public/TextToImage/IManager.php +++ b/lib/public/TextToImage/IManager.php @@ -66,6 +66,7 @@ public function scheduleTask(Task $task) : void; * @throws Exception if there was a problem inserting the task into the database * @throws PreConditionNotMetException if no provider is registered * @throws RuntimeException If the task run fail + * @since 28.0.0 */ public function runOrScheduleTask(Task $task) : void; From 9787f9dba010f17bb53e88989716c94d677c0916 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Sun, 22 Oct 2023 11:10:24 +0200 Subject: [PATCH 37/66] enh(Text2Image): Add AI settings section to text2image Signed-off-by: Marcel Klehr --- .../lib/Controller/AISettingsController.php | 2 +- .../Settings/Admin/ArtificialIntelligence.php | 11 ++++++++++ apps/settings/src/components/AdminAI.vue | 22 +++++++++++++++++++ lib/private/TextToImage/Manager.php | 6 +++++ lib/public/TextToImage/IManager.php | 6 +++++ 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/apps/settings/lib/Controller/AISettingsController.php b/apps/settings/lib/Controller/AISettingsController.php index 7f016d79c2595..8db8fa8b5bb68 100644 --- a/apps/settings/lib/Controller/AISettingsController.php +++ b/apps/settings/lib/Controller/AISettingsController.php @@ -57,7 +57,7 @@ public function __construct( * @return DataResponse */ public function update($settings) { - $keys = ['ai.stt_provider', 'ai.textprocessing_provider_preferences', 'ai.translation_provider_preferences']; + $keys = ['ai.stt_provider', 'ai.textprocessing_provider_preferences', 'ai.translation_provider_preferences', 'ai.text2image_provider']; foreach ($keys as $key) { if (!isset($settings[$key])) { continue; diff --git a/apps/settings/lib/Settings/Admin/ArtificialIntelligence.php b/apps/settings/lib/Settings/Admin/ArtificialIntelligence.php index eb1983690a5a1..4556f8e0a9d51 100644 --- a/apps/settings/lib/Settings/Admin/ArtificialIntelligence.php +++ b/apps/settings/lib/Settings/Admin/ArtificialIntelligence.php @@ -48,6 +48,7 @@ public function __construct( private ISpeechToTextManager $sttManager, private IManager $textProcessingManager, private ContainerInterface $container, + private \OCP\TextToImage\IManager $text2imageManager, ) { } @@ -101,15 +102,25 @@ public function getForm() { ]; } + $text2imageProviders = []; + foreach ($this->text2imageManager->getProviders() as $provider) { + $text2imageProviders[] = [ + 'class' => $provider::class, + 'name' => $provider->getName(), + ]; + } + $this->initialState->provideInitialState('ai-stt-providers', $sttProviders); $this->initialState->provideInitialState('ai-translation-providers', $translationProviders); $this->initialState->provideInitialState('ai-text-processing-providers', $textProcessingProviders); $this->initialState->provideInitialState('ai-text-processing-task-types', $textProcessingTaskTypes); + $this->initialState->provideInitialState('ai-text2image-providers', $text2imageProviders); $settings = [ 'ai.stt_provider' => count($sttProviders) > 0 ? $sttProviders[0]['class'] : null, 'ai.textprocessing_provider_preferences' => $textProcessingSettings, 'ai.translation_provider_preferences' => $translationPreferences, + 'ai.text2image_provider' => count($text2imageProviders) > 0 ? $text2imageProviders[0]['class'] : null, ]; foreach ($settings as $key => $defaultValue) { $value = $defaultValue; diff --git a/apps/settings/src/components/AdminAI.vue b/apps/settings/src/components/AdminAI.vue index 6a8b73d81f3dd..77755fdc0afb9 100644 --- a/apps/settings/src/components/AdminAI.vue +++ b/apps/settings/src/components/AdminAI.vue @@ -36,6 +36,24 @@ + + + + -