From b23a212072d2b17163e68011a13730d1d9fa5efd Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Tue, 17 Oct 2023 18:18:58 -0100 Subject: [PATCH] FilesMetadata Signed-off-by: Maxence Lange --- apps/files/lib/Command/Scan.php | 4 + core/Command/FilesMetadata/Get.php | 98 +++++ .../Version28000Date20231004103301.php | 80 ++++ core/register_command.php | 2 + lib/private/Files/Cache/QuerySearchHelper.php | 64 ++-- .../FilesMetadata/Event/MetadataEventBase.php | 62 +++ .../FilesMetadata/FilesMetadataManager.php | 149 ++++++++ .../Job/UpdateSingleMetadata.php | 58 +++ .../FilesMetadata/Listener/MetadataDelete.php | 60 +++ .../FilesMetadata/Listener/MetadataUpdate.php | 61 +++ .../FilesMetadata/Model/FilesMetadata.php | 353 ++++++++++++++++++ .../FilesMetadata/Model/MetadataQuery.php | 109 ++++++ .../Model/MetadataValueWrapper.php | 295 +++++++++++++++ .../Service/IndexRequestService.php | 131 +++++++ .../Service/MetadataRequestService.php | 113 ++++++ lib/private/Server.php | 5 + .../Event/MetadataBackgroundEvent.php | 40 ++ .../FilesMetadata/Event/MetadataLiveEvent.php | 64 ++++ .../Exceptions/FilesMetadataException.php | 10 + .../FilesMetadataNotFoundException.php | 8 + .../Exceptions/FilesMetadataTypeException.php | 8 + .../FilesMetadata/IFilesMetadataManager.php | 30 ++ .../FilesMetadata/Model/IFilesMetadata.php | 80 ++++ .../FilesMetadata/Model/IMetadataQuery.php | 15 + 24 files changed, 1870 insertions(+), 29 deletions(-) create mode 100644 core/Command/FilesMetadata/Get.php create mode 100644 core/Migrations/Version28000Date20231004103301.php create mode 100644 lib/private/FilesMetadata/Event/MetadataEventBase.php create mode 100644 lib/private/FilesMetadata/FilesMetadataManager.php create mode 100644 lib/private/FilesMetadata/Job/UpdateSingleMetadata.php create mode 100644 lib/private/FilesMetadata/Listener/MetadataDelete.php create mode 100644 lib/private/FilesMetadata/Listener/MetadataUpdate.php create mode 100644 lib/private/FilesMetadata/Model/FilesMetadata.php create mode 100644 lib/private/FilesMetadata/Model/MetadataQuery.php create mode 100644 lib/private/FilesMetadata/Model/MetadataValueWrapper.php create mode 100644 lib/private/FilesMetadata/Service/IndexRequestService.php create mode 100644 lib/private/FilesMetadata/Service/MetadataRequestService.php create mode 100644 lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php create mode 100644 lib/public/FilesMetadata/Event/MetadataLiveEvent.php create mode 100644 lib/public/FilesMetadata/Exceptions/FilesMetadataException.php create mode 100644 lib/public/FilesMetadata/Exceptions/FilesMetadataNotFoundException.php create mode 100644 lib/public/FilesMetadata/Exceptions/FilesMetadataTypeException.php create mode 100644 lib/public/FilesMetadata/IFilesMetadataManager.php create mode 100644 lib/public/FilesMetadata/Model/IFilesMetadata.php create mode 100644 lib/public/FilesMetadata/Model/IMetadataQuery.php diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index f5ac362719602..50f3651d3cf61 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -37,6 +37,7 @@ use OC\Core\Command\InterruptedException; use OC\DB\Connection; use OC\DB\ConnectionAdapter; +use OC\FilesMetadata\FilesMetadataManager; use OCP\Files\Events\FileCacheUpdated; use OCP\Files\Events\NodeAddedToCache; use OCP\Files\Events\NodeRemovedFromCache; @@ -69,6 +70,7 @@ public function __construct( private IUserManager $userManager, private IRootFolder $rootFolder, private MetadataManager $metadataManager, + private FilesMetadataManager $filesMetadataManager, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, ) { @@ -140,6 +142,8 @@ protected function scanFiles(string $user, string $path, bool $scanMetadata, Out if ($node instanceof File) { $this->metadataManager->generateMetadata($node, false); } + + $this->filesMetadataManager->refreshMetadata($node); } }); diff --git a/core/Command/FilesMetadata/Get.php b/core/Command/FilesMetadata/Get.php new file mode 100644 index 0000000000000..ba30afe65a671 --- /dev/null +++ b/core/Command/FilesMetadata/Get.php @@ -0,0 +1,98 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Command\FilesMetadata; + +use OC\DB\ConnectionAdapter; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IRootFolder; +use OCP\FilesMetadata\IFilesMetadataManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Get extends Command { + public function __construct( + private IRootFolder $rootFolder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct(); + } + + protected function configure() { + $this->setName('metadata:get') + ->setDescription('get stored metadata about a file, by its id') + ->addArgument( + 'fileId', + InputArgument::REQUIRED, + 'id of the file document' + ) + ->addArgument( + 'userId', + InputArgument::OPTIONAL, + 'file owner' + ) + ->addOption( + 'refresh', + '', + InputOption::VALUE_NONE, + 'refresh metadata' + ) + ->addOption( + 'reset', + '', + InputOption::VALUE_NONE, + 'refresh metadata from scratch' + ) + ->addOption( + 'background', + '', + InputOption::VALUE_NONE, + 'emulate background jobs when refreshing metadata' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $fileId = (int)$input->getArgument('fileId'); + if ($input->getOption('refresh')) { + $node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getById($fileId); + $file = $node[0]; + $metadata = $this->filesMetadataManager->refreshMetadata( + $file, + $input->getOption('background'), + $input->getOption('reset') + ); + } else { + $metadata = $this->filesMetadataManager->getMetadata($fileId); + } + + $output->writeln(json_encode($metadata, JSON_PRETTY_PRINT)); + + return 0; + } +} diff --git a/core/Migrations/Version28000Date20231004103301.php b/core/Migrations/Version28000Date20231004103301.php new file mode 100644 index 0000000000000..c6a24d770f096 --- /dev/null +++ b/core/Migrations/Version28000Date20231004103301.php @@ -0,0 +1,80 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version28000Date20231004103301 extends SimpleMigrationStep { + + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if (!$schema->hasTable('files_metadata')) { + $table = $schema->createTable('files_metadata'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15,]); + $table->addColumn('json', Types::TEXT); + $table->addColumn('sync_token', Types::STRING, ['length' => 15]); + $table->addColumn('last_update', Types::DATETIME); + + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id'], 'files_meta_fileid'); + } + + if (!$schema->hasTable('files_metadata_index')) { + $table = $schema->createTable('files_metadata_index'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15]); + $table->addColumn('meta_key', Types::STRING, ['notnull' => false, 'length' => 31]); + $table->addColumn('meta_value', Types::STRING, ['notnull' => false, 'length' => 63]); + $table->addColumn('meta_value_int', Types::BIGINT, ['notnull' => false, 'length' => 11]); +// $table->addColumn('meta_value_float', Types::FLOAT, ['notnull' => false, 'length' => 11,5]); + + $table->setPrimaryKey(['id']); + $table->addIndex(['file_id', 'meta_key', 'meta_value'], 'f_meta_index'); + $table->addIndex(['file_id', 'meta_key', 'meta_value_int'], 'f_meta_index_i'); +// $table->addIndex(['file_id', 'meta_key', 'meta_value_float'], 'f_meta_index_f'); + } + + return $schema; + } +} diff --git a/core/register_command.php b/core/register_command.php index d9e5dfcd775eb..d497a9581d1af 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -214,6 +214,8 @@ $application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager())); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class)); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class)); + + $application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class)); } else { $application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class)); } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index 15c089a0f1147..ce4c381c0e963 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -25,9 +25,11 @@ */ namespace OC\Files\Cache; +use OC\DB\ConnectionAdapter; use OC\Files\Cache\Wrapper\CacheJail; use OC\Files\Search\QueryOptimizer\QueryOptimizer; use OC\Files\Search\SearchBinaryOperator; +use OC\FilesMetadata\Model\MetadataQuery; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Cache\ICache; @@ -37,41 +39,24 @@ use OCP\Files\Mount\IMountPoint; use OCP\Files\Search\ISearchBinaryOperator; use OCP\Files\Search\ISearchQuery; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; use OCP\IGroupManager; use OCP\IUser; use Psr\Log\LoggerInterface; +use Symfony\Component\Console\Output\OutputInterface; class QuerySearchHelper { - /** @var IMimeTypeLoader */ - private $mimetypeLoader; - /** @var IDBConnection */ - private $connection; - /** @var SystemConfig */ - private $systemConfig; - private LoggerInterface $logger; - /** @var SearchBuilder */ - private $searchBuilder; - /** @var QueryOptimizer */ - private $queryOptimizer; - private IGroupManager $groupManager; - public function __construct( - IMimeTypeLoader $mimetypeLoader, - IDBConnection $connection, - SystemConfig $systemConfig, - LoggerInterface $logger, - SearchBuilder $searchBuilder, - QueryOptimizer $queryOptimizer, - IGroupManager $groupManager, + private IMimeTypeLoader $mimetypeLoader, + private IDBConnection $connection, + private SystemConfig $systemConfig, + private LoggerInterface $logger, + private SearchBuilder $searchBuilder, + private QueryOptimizer $queryOptimizer, + private IGroupManager $groupManager, + private IFilesMetadataManager $filesMetadataManager, ) { - $this->mimetypeLoader = $mimetypeLoader; - $this->connection = $connection; - $this->systemConfig = $systemConfig; - $this->logger = $logger; - $this->searchBuilder = $searchBuilder; - $this->queryOptimizer = $queryOptimizer; - $this->groupManager = $groupManager; } protected function getQueryBuilder() { @@ -144,6 +129,23 @@ protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): )); } + + protected function equipQueryForMetadata(CacheQueryBuilder $query, ISearchQuery $searchQuery): ?MetadataQuery { + // TODO: use $searchQuery to improve the query + + // init the thing + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($query, 'fc', 'fileid'); + + // get metadata aside the files + $metadataQuery->retrieveMetadata(); + + // order by metadata photo_taken + $metadataQuery->leftJoinIndex('photo_taken'); + $query->orderBy($metadataQuery->getMetadataValueIntField(), 'desc'); + + return $metadataQuery; + } + /** * Perform a file system search in multiple caches * @@ -175,6 +177,7 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array $query = $builder->selectFileCache('file', false); $requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation()); + if (in_array('systemtag', $requestedFields)) { $this->equipQueryForSystemTags($query, $this->requireUser($searchQuery)); } @@ -182,13 +185,16 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); } + $metadataQuery = $this->equipQueryForMetadata($query, $searchQuery); $this->applySearchConstraints($query, $searchQuery, $caches); $result = $query->execute(); $files = $result->fetchAll(); - $rawEntries = array_map(function (array $data) { - return Cache::cacheEntryFromData($data, $this->mimetypeLoader); + $rawEntries = array_map(function (array $data) use ($metadataQuery) { + $entry = Cache::cacheEntryFromData($data, $this->mimetypeLoader); + $entry['metadata'] = $metadataQuery?->extractMetadata($data); + return $entry; }, $files); $result->closeCursor(); diff --git a/lib/private/FilesMetadata/Event/MetadataEventBase.php b/lib/private/FilesMetadata/Event/MetadataEventBase.php new file mode 100644 index 0000000000000..758ce96d848ca --- /dev/null +++ b/lib/private/FilesMetadata/Event/MetadataEventBase.php @@ -0,0 +1,62 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Event; + +use OCP\Files\Node; +use OCP\EventDispatcher\Event; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataEventBase extends Event { + + public function __construct( + protected Node $node, + protected IFilesMetadata $metadata + ) { + parent::__construct(); + } + + /** + * returns id of the file + * + * @return Node + * @since 28.0.0 + */ + public function getNode(): Node { + return $this->node; + } + + /** + * returns Metadata model linked to file id, with already known metadata from the database. + * If the object is modified using its setters, metadata are updated in database at the end of the event. + * + * @return IFilesMetadata + * @since 28.0.0 + */ + public function getMetadata(): IFilesMetadata { + return $this->metadata; + } +} diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php new file mode 100644 index 0000000000000..19d8853b28c23 --- /dev/null +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -0,0 +1,149 @@ +metadataRequestService->getMetadataFromFileId($fileId); + } + + + public function refreshMetadata( + Node $node, + bool $asBackgroundJob = false, + bool $fromScratch = false + ): IFilesMetadata { + $metadata = null; + if (!$fromScratch) { + try { + $metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId()); + } catch (FilesMetadataNotFoundException $e) { + } + } + + if (null === $metadata) { + $metadata = new FilesMetadata($node->getId(), true); + } + + if ($asBackgroundJob) { + $event = new MetadataBackgroundEvent($node, $metadata); + } else { + $event = new MetadataLiveEvent($node, $metadata); + } + + $this->eventDispatcher->dispatchTyped($event); + $this->saveMetadata($event->getMetadata()); + + // if requested, we add a new job for next cron to refresh metadata out of main thread + if ($event instanceof MetadataLiveEvent && $event->isRunAsBackgroundJobRequested()) { + $this->jobList->add(UpdateSingleMetadata::class, [$node->getOwner()->getUID(), $node->getId()]); + } + + return $metadata; + } + + /** + * @param IFilesMetadata $filesMetadata + * + * @return void + */ + public function saveMetadata(IFilesMetadata $filesMetadata): void { + if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) { + return; + } + + try { + // if update request changed no rows, means that new entry is needed, or sync_token not valid anymore + $updated = $this->metadataRequestService->updateMetadata($filesMetadata); + if ($updated === 0) { + $this->metadataRequestService->store($filesMetadata); + } + } catch (\OCP\DB\Exception $e) { + // if duplicate, only means a desync during update. cancel update process. + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + $this->logger->warning( + 'issue while saveMetadata', ['exception' => $e, 'metadata' => $filesMetadata] + ); + } + + return; + } + +// $this->removeDeprecatedMetadata($filesMetadata); + foreach ($filesMetadata->getIndexes() as $index) { + try { + $this->indexRequestService->updateIndex($filesMetadata, $index); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + } + + public function deleteMetadata(int $fileId): void { + try { + $this->metadataRequestService->dropMetadata($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + + try { + $this->indexRequestService->dropIndex($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + } + + public function getMetadataQuery( + IQueryBuilder $qb, + string $fileTableAlias, + string $fileIdField + ): IMetadataQuery { + return new MetadataQuery($qb, $fileTableAlias, $fileIdField); + } + + public static function loadListeners(IEventDispatcher $eventDispatcher): void { + $eventDispatcher->addServiceListener(NodeCreatedEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeDeletedEvent::class, MetadataDelete::class); + } +} diff --git a/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php new file mode 100644 index 0000000000000..e25acb8ec5cc2 --- /dev/null +++ b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php @@ -0,0 +1,58 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Job; + +use OC\FilesMetadata\FilesMetadataManager; +use OC\User\NoUserException; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; +use OCP\Files\IRootFolder; +use OCP\Files\NotPermittedException; + +class UpdateSingleMetadata extends QueuedJob { + public function __construct( + ITimeFactory $time, + private IRootFolder $rootFolder, + private FilesMetadataManager $filesMetadataManager, + ) { + parent::__construct($time); + } + + protected function run($argument) { + [$userId, $fileId] = $argument; + + try { + // TODO: is there a way to get Node without $userId ? + $node = $this->rootFolder->getUserFolder($userId)->getById($fileId); + if (count($node) > 0) { + $file = array_shift($node); + $this->filesMetadataManager->refreshMetadata($file, true); + } + } catch (NotPermittedException |NoUserException $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataDelete.php b/lib/private/FilesMetadata/Listener/MetadataDelete.php new file mode 100644 index 0000000000000..ccc52d489cf33 --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataDelete.php @@ -0,0 +1,60 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\FilesMetadata\IFilesMetadataManager; + +/** + * Handle file deletion event and remove stored metadata related to the deleted file + */ +class MetadataDelete implements IEventListener { + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeDeletedEvent)) { + return; + } + + try { + $nodeId = (int)$event->getNode()->getId(); + if ($nodeId > 0) { + $this->filesMetadataManager->deleteMetadata($nodeId); + } + } catch (Exception $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataUpdate.php b/lib/private/FilesMetadata/Listener/MetadataUpdate.php new file mode 100644 index 0000000000000..3e1d8ef9753cf --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataUpdate.php @@ -0,0 +1,61 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeCreatedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\FilesMetadata\IFilesMetadataManager; + +/** + * Handle file creation/modification events and initiate a new event related to the created/edited file. + * The generated new event is broadcast in order to obtain file related metadata from other apps. + * metadata will be stored in database. + */ +class MetadataUpdate implements IEventListener { + + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeCreatedEvent) && !($event instanceof NodeWrittenEvent)) { + return; + } + + try { + $this->filesMetadataManager->refreshMetadata($event->getNode()); + } catch (Exception $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php new file mode 100644 index 0000000000000..4460ac1e4d5c3 --- /dev/null +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -0,0 +1,353 @@ + */ + private array $metadata = []; + private int $lastUpdate = 0; + private string $syncToken = ''; + + public function __construct( + private int $fileId = 0, + private bool $updated = false + ) { + } + + public function getFileId(): int { + return $this->fileId; + } + + /** + * @param array $data + * + * @return IFilesMetadata + */ + public function import(array $data): IFilesMetadata { + foreach ($data as $k => $v) { + $valueWrapper = new MetadataValueWrapper(); + $this->metadata[$k] = $valueWrapper->import($v); + } + $this->updated = false; + + return $this; + } + + + /** + * import from database using the json field. + * + * if using aliases (ie. myalias_json), use $prefix='myalias_' + * + * @param array $data + * @param string $prefix + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + */ + public function importFromDatabase(array $data, string $prefix = ''): IFilesMetadata { + try { + return $this->import( + json_decode($data[$prefix . 'json'] ?? '[]', + true, + 512, + JSON_THROW_ON_ERROR) + ); + } catch (JsonException $e) { + throw new FilesMetadataNotFoundException(); + } + } + + + public function updated(): bool { + return $this->updated; + } + + public function lastUpdateTimestamp(): int { + return $this->lastUpdate; + } + + public function getSyncToken(): string { + return $this->syncToken; + } + + public function hasKey(string $needle): bool { + return (in_array($needle, $this->getKeys())); + } + + public function getKeys(): array { + return array_keys($this->metadata); + } + + /** + * @return string[] + */ + public function getIndexes(): array { + $indexes = []; + foreach ($this->getKeys() as $key) { + if ($this->metadata[$key]->isIndexed()) { + $indexes[] = $key; + } + } + + return $indexes; + } + + /** + * @param string $key + * + * @return string + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function get(string $key): string { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueString(); + } + + /** + * @param string $key + * + * @return int + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getInt(string $key): int { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueInt(); + } + + /** + * @param string $key + * + * @return float + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getFloat(string $key): float { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueFloat(); + } + + /** + * @param string $key + * + * @return bool + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getBool(string $key): bool { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueBool(); + } + + /** + * @param string $key + * + * @return array + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getArray(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueArray(); + } + + /** + * @param string $key + * + * @return string[] + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getStringList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueStringList(); + } + + /** + * @param string $key + * + * @return int[] + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getIntList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueIntList(); + } + + public function getType(string $key): string { + return $this->metadata[$key]->getType(); + } + + public function set(string $key, string $value, bool $index = false): IFilesMetadata { + try { + if ($this->get($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value and index have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING); + $this->updated = true; + $this->metadata[$key] = $meta->setValueString($value)->setIndexed($index); + + return $this; + } + + public function setInt(string $key, int $value, bool $index = false): IFilesMetadata { + try { + if ($this->getInt($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_INT); + $this->metadata[$key] = $meta->setValueInt($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setFloat(string $key, float $value, bool $index = false): IFilesMetadata { + try { + if ($this->getFloat($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_FLOAT); + $this->metadata[$key] = $meta->setValueFloat($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setBool(string $key, bool $value): IFilesMetadata { + try { + if ($this->getBool($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_BOOL); + $this->metadata[$key] = $meta->setValueBool($value); + $this->updated = true; + + return $this; + } + + public function setArray(string $key, array $value): IFilesMetadata { + try { + if ($this->getArray($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_ARRAY); + $this->metadata[$key] = $meta->setValueArray($value); + $this->updated = true; + + return $this; + } + + + public function setStringList(string $key, array $values, bool $index = false): IFilesMetadata { + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $meta->setValueStringList($values)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setIntList(string $key, array $values, bool $index = false): IFilesMetadata { + $valueWrapper = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $valueWrapper->setValueIntList($values)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function unset(string $key): IFilesMetadata { + unset($this->metadata[$key]); + $this->updated = true; + + return $this; + } + + public function removeStartsWith(string $keyPrefix): IFilesMetadata { + if ($keyPrefix === '') { + return $this; + } + + foreach ($this->getKeys() as $key) { + if (str_starts_with($key, $keyPrefix)) { + $this->unset($key); + } + } + + return $this; + } + + /** + * @param string $key + * + * @return MetadataValueWrapper + * @throws FilesMetadataNotFoundException + */ + public function getValueWrapper(string $key): MetadataValueWrapper { + if (!$this->hasKey($key)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]; + } + + + public function jsonSerialize(): array { + return $this->metadata; + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataQuery.php b/lib/private/FilesMetadata/Model/MetadataQuery.php new file mode 100644 index 0000000000000..c07e1a686ca09 --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataQuery.php @@ -0,0 +1,109 @@ +queryBuilder->expr(); + $andX = $expr->andX($expr->eq($this->aliasIndex . '.file_id', $this->fileTableAlias . '.' . $this->fileIdField)); + + if ('' !== $metadataKey) { + $andX->add($expr->eq($this->getMetadataKeyField(), $this->queryBuilder->createNamedParameter($metadataKey))); + } + + $this->queryBuilder->leftJoin( + $this->fileTableAlias, + IndexRequestService::TABLE_METADATA_INDEX, + $this->aliasIndex, + $andX + ); + } + + + /** + * left join the metadata table to include a select of the stored json to the query + */ + public function retrieveMetadata(): void { + $this->queryBuilder->selectAlias($this->alias . '.json', 'meta_json'); + $this->queryBuilder->leftJoin( + $this->fileTableAlias, MetadataRequestService::TABLE_METADATA, $this->alias, + $this->queryBuilder->expr()->eq($this->fileTableAlias . '.' . $this->fileIdField, $this->alias . '.file_id') + ); + } + + public function enforceMetadataKey(string $metadataKey): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataKeyField(), + $this->queryBuilder->createNamedParameter($metadataKey) + ) + ); + } + + public function enforceMetadataValue(string $value): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataKeyField(), + $this->queryBuilder->createNamedParameter($value) + ) + ); + } + + public function enforceMetadataValueInt(int $value): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataValueIntField(), + $this->queryBuilder->createNamedParameter($value, IQueryBuilder::PARAM_INT) + ) + ); + } + + public function getMetadataKeyField(): string { + return $this->aliasIndex . '.meta_key'; + } + + public function getMetadataValueField(): string { + return $this->aliasIndex . '.meta_value'; + } + + public function getMetadataValueIntField(): string { + return $this->aliasIndex . '.meta_value_int'; + } + + public function extractMetadata(array $data): IFilesMetadata { + $fileId = 1; // from $data, in an alias field generated by limitToSingleMetadata() + + $metadata = new FilesMetadata($fileId); + $metadata->importFromDatabase($data, $this->alias . '_'); + + return $metadata; + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataValueWrapper.php b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php new file mode 100644 index 0000000000000..132a5a8328d4a --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php @@ -0,0 +1,295 @@ +type = $type; + } + + public function setType(string $type): self { + $this->type = $type; + return $this; + } + + public function getType(): string { + return $this->type; + } + + public function isType(string $type): bool { + return (strtolower($type) === strtolower($this->type)); + } + + /** + * confirm stored value exists and is typed as requested + * @param string $type + * + * @return $this + * @throws FilesMetadataTypeException + */ + public function confirmType(string $type): self { + if (!$this->isType($type)) { + throw new FilesMetadataTypeException('type is \'' . $this->getType() . '\', expecting \'' . $type . '\''); + } + + return $this; + } + + /** + * @param string $value + * + * @return $this + */ + public function setValueString(string $value): self { + if ($this->isType(self::TYPE_STRING)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param int $value + * + * @return $this + */ + public function setValueInt(int $value): self { + if ($this->isType(self::TYPE_INT)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param float $value + * + * @return $this + */ + public function setValueFloat(float $value): self { + if ($this->isType(self::TYPE_FLOAT)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param bool $value + * + * @return $this + */ + public function setValueBool(bool $value): self { + if ($this->isType(self::TYPE_BOOL)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param array $value + * + * @return $this + */ + public function setValueArray(array $value): self { + if ($this->isType(self::TYPE_ARRAY)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param string[] $value + * + * @return $this + */ + public function setValueStringList(array $value): self { + if ($this->isType(self::TYPE_STRING_LIST)) { + // TODO confirm value is an array or string ? + $this->value = $value; + } + + return $this; + } + + /** + * @param int[] $value + * + * @return $this + */ + public function setValueIntList(array $value): self { + if ($this->isType(self::TYPE_INT_LIST)) { + // TODO confirm value is an array of int ? + $this->value = $value; + } + + return $this; + } + + + /** + * @return string + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueString(): string { + $this->confirmType(self::TYPE_STRING); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (string) $this->value; + } + + /** + * @return int + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueInt(): int { + $this->confirmType(self::TYPE_INT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (int) $this->value; + } + + /** + * @return float + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueFloat(): float { + $this->confirmType(self::TYPE_FLOAT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (float) $this->value; + } + + /** + * @return bool + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueBool(): bool { + $this->confirmType(self::TYPE_BOOL); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (bool) $this->value; + } + + /** + * @return array + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueArray(): array { + $this->confirmType(self::TYPE_ARRAY); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + /** + * @return string[] + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueStringList(): array { + $this->confirmType(self::TYPE_STRING_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + /** + * @return array + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueIntList(): array { + $this->confirmType(self::TYPE_INT_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + + public function setIndexed(bool $indexed): self { + $this->indexed = $indexed; + + return $this; + } + + public function isIndexed(): bool { + return $this->indexed; + } + + public function import(array $data): self { + $this->value = $data['value'] ?? null; + $this->setType($data['type'] ?? ''); + $this->setIndexed($data['indexed'] ?? false); + + return $this; + } + + public function jsonSerialize(): array { + return [ + 'value' => $this->value, + 'type' => $this->getType(), + 'indexed' => $this->isIndexed() + ]; + } +} diff --git a/lib/private/FilesMetadata/Service/IndexRequestService.php b/lib/private/FilesMetadata/Service/IndexRequestService.php new file mode 100644 index 0000000000000..6804625de3235 --- /dev/null +++ b/lib/private/FilesMetadata/Service/IndexRequestService.php @@ -0,0 +1,131 @@ +getFileId(); + + /** + * might look harsh, but a lot simpler than comparing current indexed data, as we can expect + * conflict with a change of types. + * We assume that each time one random metadata were modified we can drop all index for this + * key and recreate them + */ + $this->dropIndex($fileId, $key); + + try { + match ($filesMetadata->getType($key)) { + MetadataValueWrapper::TYPE_STRING + => $this->insertIndexString($fileId, $key, $filesMetadata->get($key)), + MetadataValueWrapper::TYPE_INT + => $this->insertIndexInt($fileId, $key, $filesMetadata->getInt($key)), + MetadataValueWrapper::TYPE_STRING_LIST + => $this->insertIndexStringList($fileId, $key, $filesMetadata->getStringList($key)), + MetadataValueWrapper::TYPE_INT_LIST + => $this->insertIndexIntList($fileId, $key, $filesMetadata->getIntList($key)) + }; + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + $this->logger->warning('...'); + } + } + + + private function insertIndexString(int $fileId, string $key, string $value): void { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value', $qb->createNamedParameter($value)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + + public function insertIndexInt(int $fileId, string $key, int $value): void { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_int', $qb->createNamedParameter($value, IQueryBuilder::PARAM_INT)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + + /** + * @param int $fileId + * @param string $key + * @param string[] $values + * + * @return void + */ + public function insertIndexStringList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexString($fileId, $key, $value); + } + } + + /** + * @param int $fileId + * @param string $key + * @param int[] $values + * + * @return void + */ + public function insertIndexIntList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexInt($fileId, $key, $value); + } + } + + /** + * @param int $fileId + * @param string $key + * + * @return void + * @throws Exception + */ + public function dropIndex(int $fileId, string $key = ''): void { + $qb = $this->dbConnection->getQueryBuilder(); + $expr = $qb->expr(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($expr->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + if ($key !== '') { + $qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key))); + } + + $qb->executeStatement(); + } +} diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php new file mode 100644 index 0000000000000..0c82d80b760a8 --- /dev/null +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -0,0 +1,113 @@ +dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA) + ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) + ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->setValue('sync_token', $qb->createNamedParameter($filesMetadata->getSyncToken())) + ->setValue('last_update', $qb->createFunction('NOW()')); + $qb->executeStatement(); + } + + /** + * @param int $fileId + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + */ + public function getMetadataFromFileId(int $fileId): IFilesMetadata { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('json')->from(self::TABLE_METADATA); + $qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + $data = $result->fetch(); + $result->closeCursor(); + } catch (Exception $e) { + $this->logger->warning('exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId]); + throw new FilesMetadataNotFoundException(); + } + + if ($data === false) { + throw new FilesMetadataNotFoundException(); + } + + $metadata = new FilesMetadata($fileId); + $metadata->importFromDatabase($data); + + return $metadata; + } + + + /** + * @param int $fileId + * + * @return void + * @throws Exception + */ + public function dropMetadata(int $fileId): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + } + + private function removeDeprecatedMetadata(IFilesMetadata $filesMetadata): void { + // TODO delete aussi les index generate a partir d'une string[] + + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->notIn('file_id', $filesMetadata->getIndexes(), IQueryBuilder::PARAM_STR_ARRAY)); + $qb->executeStatement(); + } + + + /** + * @param IFilesMetadata $filesMetadata + * + * @return bool + * @throws Exception + */ + public function updateMetadata(IFilesMetadata $filesMetadata): int { + // TODO check sync_token on update + $qb = $this->dbConnection->getQueryBuilder(); + $qb->update(self::TABLE_METADATA) + ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->set('sync_token', $qb->createNamedParameter('abc')) + ->set('last_update', $qb->createFunction('NOW()')) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))); + + return $qb->executeStatement(); + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index 949a7ccfd3f4d..fd15e064b57a1 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -102,6 +102,7 @@ use OC\Files\Template\TemplateManager; use OC\Files\Type\Loader; use OC\Files\View; +use OC\FilesMetadata\FilesMetadataManager; use OC\FullTextSearch\FullTextSearchManager; use OC\Http\Client\ClientService; use OC\Http\Client\NegativeDnsCache; @@ -194,6 +195,7 @@ use OCP\Files\Mount\IMountManager; use OCP\Files\Storage\IStorageFactory; use OCP\Files\Template\ITemplateManager; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FullTextSearch\IFullTextSearchManager; use OCP\GlobalScale\IConfig; use OCP\Group\ISubAdmin; @@ -1397,6 +1399,7 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(\OCP\Dashboard\IManager::class, \OC\Dashboard\Manager::class); $this->registerAlias(IFullTextSearchManager::class, FullTextSearchManager::class); + $this->registerAlias(IFilesMetadataManager::class, FilesMetadataManager::class); $this->registerAlias(ISubAdmin::class, SubAdmin::class); @@ -1470,6 +1473,8 @@ private function connectDispatcher(): void { $eventDispatcher->addServiceListener(PostLoginEvent::class, UserLoggedInListener::class); $eventDispatcher->addServiceListener(UserChangedEvent::class, UserChangedListener::class); $eventDispatcher->addServiceListener(BeforeUserDeletedEvent::class, BeforeUserDeletedListener::class); + + FilesMetadataManager::loadListeners($eventDispatcher); } /** diff --git a/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php b/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php new file mode 100644 index 0000000000000..320d98c8edfe1 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataBackgroundEvent.php @@ -0,0 +1,40 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\FilesMetadata\Event; + +use OC\FilesMetadata\Event\MetadataEventBase; +use OCP\Files\Node; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataBackgroundEvent extends MetadataEventBase { + public function __construct( + Node $node, + IFilesMetadata $metadata + ) { + parent::__construct($node, $metadata); + } +} diff --git a/lib/public/FilesMetadata/Event/MetadataLiveEvent.php b/lib/public/FilesMetadata/Event/MetadataLiveEvent.php new file mode 100644 index 0000000000000..c9c2306368566 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataLiveEvent.php @@ -0,0 +1,64 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\FilesMetadata\Event; + +use OC\FilesMetadata\Event\MetadataEventBase; +use OCP\Files\Node; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataLiveEvent extends MetadataEventBase { + private bool $runAsBackgroundJob = false; + + public function __construct( + Node $node, + IFilesMetadata $metadata + ) { + parent::__construct($node, $metadata); + } + + /** + * If an app prefer to update metadata on a background job, instead of + * live process, just call this method. + * A new event will be generated on next cron tick. + * + * @return void + * @since 28.0.0 + */ + public function requestBackgroundJob(): void { + $this->runAsBackgroundJob = true; + } + + /** + * return true if any app that catch this event requested a re-run as background job + * + * @return bool + * @since 28.0.0 + */ + public function isRunAsBackgroundJobRequested(): bool { + return $this->runAsBackgroundJob; + } +} diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php new file mode 100644 index 0000000000000..93685e22becf7 --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php @@ -0,0 +1,10 @@ +