diff --git a/core/Command/FilesMetadata/Get.php b/core/Command/FilesMetadata/Get.php new file mode 100644 index 0000000000000..c9b032c3e591e --- /dev/null +++ b/core/Command/FilesMetadata/Get.php @@ -0,0 +1,97 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Command\FilesMetadata; + +use OCP\Files\IRootFolder; +use OCP\FilesMetadata\IFilesMetadataManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Get extends Command { + public function __construct( + private IRootFolder $rootFolder, + private IFilesMetadataManager $filesMetadataManager, + ) { + parent::__construct(); + } + + protected function configure() { + $this->setName('metadata:get') + ->setDescription('get stored metadata about a file, by its id') + ->addArgument( + 'fileId', + InputArgument::REQUIRED, + 'id of the file document' + ) + ->addArgument( + 'userId', + InputArgument::REQUIRED, + 'file owner' + ) + ->addOption( + 'refresh', + '', + InputOption::VALUE_NONE, + 'refresh metadata' + ) + ->addOption( + 'reset', + '', + InputOption::VALUE_NONE, + 'refresh metadata from scratch' + ) + ->addOption( + 'background', + '', + InputOption::VALUE_NONE, + 'emulate background jobs when refreshing metadata' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $fileId = (int)$input->getArgument('fileId'); + if ($input->getOption('refresh')) { + $node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getById($fileId); + $file = $node[0]; + $metadata = $this->filesMetadataManager->refreshMetadata( + $file, + $input->getOption('background'), + $input->getOption('reset') + ); + } else { + $metadata = $this->filesMetadataManager->getMetadata($fileId); + } + + $output->writeln(json_encode($metadata, JSON_PRETTY_PRINT)); + + return 0; + } + +} diff --git a/core/Migrations/Version28000Date20231004103301.php b/core/Migrations/Version28000Date20231004103301.php new file mode 100644 index 0000000000000..c6a24d770f096 --- /dev/null +++ b/core/Migrations/Version28000Date20231004103301.php @@ -0,0 +1,80 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\Migrations; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version28000Date20231004103301 extends SimpleMigrationStep { + + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if (!$schema->hasTable('files_metadata')) { + $table = $schema->createTable('files_metadata'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15,]); + $table->addColumn('json', Types::TEXT); + $table->addColumn('sync_token', Types::STRING, ['length' => 15]); + $table->addColumn('last_update', Types::DATETIME); + + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id'], 'files_meta_fileid'); + } + + if (!$schema->hasTable('files_metadata_index')) { + $table = $schema->createTable('files_metadata_index'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + 'length' => 15, + 'unsigned' => true, + ]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15]); + $table->addColumn('meta_key', Types::STRING, ['notnull' => false, 'length' => 31]); + $table->addColumn('meta_value', Types::STRING, ['notnull' => false, 'length' => 63]); + $table->addColumn('meta_value_int', Types::BIGINT, ['notnull' => false, 'length' => 11]); +// $table->addColumn('meta_value_float', Types::FLOAT, ['notnull' => false, 'length' => 11,5]); + + $table->setPrimaryKey(['id']); + $table->addIndex(['file_id', 'meta_key', 'meta_value'], 'f_meta_index'); + $table->addIndex(['file_id', 'meta_key', 'meta_value_int'], 'f_meta_index_i'); +// $table->addIndex(['file_id', 'meta_key', 'meta_value_float'], 'f_meta_index_f'); + } + + return $schema; + } +} diff --git a/core/register_command.php b/core/register_command.php index d9e5dfcd775eb..d497a9581d1af 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -214,6 +214,8 @@ $application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager())); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class)); $application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class)); + + $application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class)); } else { $application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class)); } diff --git a/lib/private/FilesMetadata/Event/MetadataEventBase.php b/lib/private/FilesMetadata/Event/MetadataEventBase.php new file mode 100644 index 0000000000000..758ce96d848ca --- /dev/null +++ b/lib/private/FilesMetadata/Event/MetadataEventBase.php @@ -0,0 +1,62 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Event; + +use OCP\Files\Node; +use OCP\EventDispatcher\Event; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataEventBase extends Event { + + public function __construct( + protected Node $node, + protected IFilesMetadata $metadata + ) { + parent::__construct(); + } + + /** + * returns id of the file + * + * @return Node + * @since 28.0.0 + */ + public function getNode(): Node { + return $this->node; + } + + /** + * returns Metadata model linked to file id, with already known metadata from the database. + * If the object is modified using its setters, metadata are updated in database at the end of the event. + * + * @return IFilesMetadata + * @since 28.0.0 + */ + public function getMetadata(): IFilesMetadata { + return $this->metadata; + } +} diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php new file mode 100644 index 0000000000000..3aa976f4e21de --- /dev/null +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -0,0 +1,159 @@ +metadataRequestService->getMetadataFromFileId($fileId); + } + + + public function refreshMetadata( + Node $node, + bool $asBackgroundJob = false, + bool $fromScratch = false + ): IFilesMetadata { + $metadata = null; + if (!$fromScratch) { + try { + $metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId()); + } catch (FilesMetadataNotFoundException $e) { + } + } + + if (null === $metadata) { + $metadata = new FilesMetadata($node->getId(), true); + } + + if ($asBackgroundJob) { + $event = new MetadataEventBackgroundEvent($node, $metadata); + } else { + $event = new MetadataEventLiveEvent($node, $metadata); + } + + $this->eventDispatcher->dispatchTyped($event); + $this->saveMetadata($event->getMetadata()); + + // if requested, we add a new job for next cron to refresh metadata out of main thread + if ($event instanceof MetadataEventLiveEvent && $event->isRunAsBackgroundJobRequested()) { + $this->jobList->add(UpdateSingleMetadata::class, [$node->getOwner()->getUID(), $node->getId()]); + } + + return $metadata; + } + + /** + * @param IFilesMetadata $filesMetadata + * + * @return void + */ + public function saveMetadata(IFilesMetadata $filesMetadata): void { + if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) { + return; + } + + try { + // if update request changed no rows, means that new entry is needed, or sync_token not valid anymore + $updated = $this->metadataRequestService->updateMetadata($filesMetadata); + if ($updated === 0) { + $this->metadataRequestService->store($filesMetadata); + } + } catch (\OCP\DB\Exception $e) { + // if duplicate, only means a desync during update. cancel update process. + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + $this->logger->warning( + 'issue while saveMetadata', ['exception' => $e, 'metadata' => $filesMetadata] + ); + } + + return; + } + +// $this->removeDeprecatedMetadata($filesMetadata); + foreach ($filesMetadata->getIndexes() as $index) { + try { + $this->indexRequestService->updateIndex($filesMetadata, $index); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + } + + + public function deleteMetadata(int $fileId): void { + try { + $this->metadataRequestService->dropMetadata($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + + try { + $this->indexRequestService->dropIndex($fileId); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]); + } + } + + public function getMetadataQuery( + IQueryBuilder $qb, + string $fileTableAlias, + string $fileIdField + ): IMetadataQuery { + $metadataQuery = new MetadataQuery($qb); + $metadataQuery->leftJoinIndex($fileTableAlias, $fileIdField); + + return $metadataQuery; + } + + + + + + public static function loadListeners(IEventDispatcher $eventDispatcher): void { + $eventDispatcher->addServiceListener(NodeCreatedEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); + $eventDispatcher->addServiceListener(NodeDeletedEvent::class, MetadataDelete::class); + } +} diff --git a/lib/private/FilesMetadata/FilesMetadataQueryHelper.php b/lib/private/FilesMetadata/FilesMetadataQueryHelper.php new file mode 100644 index 0000000000000..3273ff8e28d03 --- /dev/null +++ b/lib/private/FilesMetadata/FilesMetadataQueryHelper.php @@ -0,0 +1,106 @@ +expr(); + if ($selectMetadata) { + $qb->selectAlias('meta.json', 'meta_json'); + } + + $qb->leftJoin( + $fileTableAlias, IndexRequestService::TABLE_METADATA_INDEX, 'meta_index', + $expr->eq('meta_index.file_id', $fileTableAlias . '.' . $fileIdField) + ) + ->andwhere( + $expr->andX( + $expr->eq('meta_index.meta_key', $qb->createNamedParameter($metadataKey)), + $expr->eq('meta_index.meta_value', $qb->createNamedParameter($metadataValue)) + ) + ); + + if ($selectMetadata) { + $qb->leftJoin( + 'meta_index', MetadataRequestService::TABLE_METADATA, 'meta', + $expr->eq('meta_index.file_id', 'meta.file_id') + ); + } + } + + + public function limitToSingleMetadataInt( + IQueryBuilder $qb, + string $fileTableAlias, + string $fileIdField, + string $metadataKey, + int $metadataValue, + bool $selectMetadata = false, + string $exprType = 'eq' + ): void { + $expr = $qb->expr(); + if ($selectMetadata) { + $qb->selectAlias('meta.json', 'meta_json'); + } + + $qb->leftJoin( + $fileTableAlias, IndexRequestService::TABLE_METADATA_INDEX, 'meta_index', + $expr->eq('meta_index.file_id', $fileTableAlias . '.' . $fileIdField) + ) + ->andwhere( + $expr->andX( + $expr->eq('meta_index.meta_key', $qb->createNamedParameter($metadataKey)), + $expr->$exprType('meta_index.meta_value', $qb->createNamedParameter($metadataValue, IQueryBuilder::PARAM_INT)) + ) + ); + + if ($selectMetadata) { + $qb->leftJoin( + 'meta_index', MetadataRequestService::TABLE_METADATA, 'meta', + $expr->eq('meta_index.file_id', 'meta.file_id') + ); + } + } + + /** + * @param array $data + * + * @return IFilesMetadata + */ + public function extractMetadata(array $data): IFilesMetadata { + $json = '[]'; // from $data, in an alias field generated by limitToSingleMetadata() + $fileId = 1; // from $data, in an alias field generated by limitToSingleMetadata() + + $metadata = new FilesMetadata($fileId); + $metadata->import($json); + + return $metadata; + } +} diff --git a/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php new file mode 100644 index 0000000000000..e25acb8ec5cc2 --- /dev/null +++ b/lib/private/FilesMetadata/Job/UpdateSingleMetadata.php @@ -0,0 +1,58 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Job; + +use OC\FilesMetadata\FilesMetadataManager; +use OC\User\NoUserException; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; +use OCP\Files\IRootFolder; +use OCP\Files\NotPermittedException; + +class UpdateSingleMetadata extends QueuedJob { + public function __construct( + ITimeFactory $time, + private IRootFolder $rootFolder, + private FilesMetadataManager $filesMetadataManager, + ) { + parent::__construct($time); + } + + protected function run($argument) { + [$userId, $fileId] = $argument; + + try { + // TODO: is there a way to get Node without $userId ? + $node = $this->rootFolder->getUserFolder($userId)->getById($fileId); + if (count($node) > 0) { + $file = array_shift($node); + $this->filesMetadataManager->refreshMetadata($file, true); + } + } catch (NotPermittedException |NoUserException $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataDelete.php b/lib/private/FilesMetadata/Listener/MetadataDelete.php new file mode 100644 index 0000000000000..ccc52d489cf33 --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataDelete.php @@ -0,0 +1,60 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\FilesMetadata\IFilesMetadataManager; + +/** + * Handle file deletion event and remove stored metadata related to the deleted file + */ +class MetadataDelete implements IEventListener { + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeDeletedEvent)) { + return; + } + + try { + $nodeId = (int)$event->getNode()->getId(); + if ($nodeId > 0) { + $this->filesMetadataManager->deleteMetadata($nodeId); + } + } catch (Exception $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Listener/MetadataUpdate.php b/lib/private/FilesMetadata/Listener/MetadataUpdate.php new file mode 100644 index 0000000000000..3e1d8ef9753cf --- /dev/null +++ b/lib/private/FilesMetadata/Listener/MetadataUpdate.php @@ -0,0 +1,61 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\FilesMetadata\Listener; + +use Exception; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeCreatedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; +use OCP\FilesMetadata\IFilesMetadataManager; + +/** + * Handle file creation/modification events and initiate a new event related to the created/edited file. + * The generated new event is broadcast in order to obtain file related metadata from other apps. + * metadata will be stored in database. + */ +class MetadataUpdate implements IEventListener { + + public function __construct( + private IFilesMetadataManager $filesMetadataManager, + ) { + } + + /** + * @param Event $event + */ + public function handle(Event $event): void { + if (!($event instanceof NodeCreatedEvent) && !($event instanceof NodeWrittenEvent)) { + return; + } + + try { + $this->filesMetadataManager->refreshMetadata($event->getNode()); + } catch (Exception $e) { + } + } +} diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php new file mode 100644 index 0000000000000..9cf023d0274cb --- /dev/null +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -0,0 +1,326 @@ + */ + private array $metadata = []; + private int $lastUpdate = 0; + private string $syncToken = ''; + + public function __construct( + private int $fileId = 0, + private bool $updated = false + ) { + } + + public function getFileId(): int { + return $this->fileId; + } + + /** + * @param array $data + * + * @return IFilesMetadata + */ + public function import(array $data): IFilesMetadata { + foreach ($data as $k => $v) { + $valueWrapper = new MetadataValueWrapper(); + $this->metadata[$k] = $valueWrapper->import($v); + } + $this->updated = false; + + return $this; + } + + public function updated(): bool { + return $this->updated; + } + + public function lastUpdateTimestamp(): int { + return $this->lastUpdate; + } + + public function getSyncToken(): string { + return $this->syncToken; + } + + public function hasKey(string $needle): bool { + return (in_array($needle, $this->getKeys())); + } + + public function getKeys(): array { + return array_keys($this->metadata); + } + + /** + * @return string[] + */ + public function getIndexes(): array { + $indexes = []; + foreach ($this->getKeys() as $key) { + if ($this->metadata[$key]->isIndexed()) { + $indexes[] = $key; + } + } + + return $indexes; + } + + /** + * @param string $key + * + * @return string + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function get(string $key): string { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueString(); + } + + /** + * @param string $key + * + * @return int + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getInt(string $key): int { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueInt(); + } + + /** + * @param string $key + * + * @return float + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getFloat(string $key): float { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueFloat(); + } + + /** + * @param string $key + * + * @return bool + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getBool(string $key): bool { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueBool(); + } + + /** + * @param string $key + * + * @return array + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getArray(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueArray(); + } + + /** + * @param string $key + * + * @return string[] + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getStringList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueStringList(); + } + + /** + * @param string $key + * + * @return int[] + * @throws FilesMetadataNotFoundException + * @throws FilesMetadataTypeException + */ + public function getIntList(string $key): array { + if (!array_key_exists($key, $this->metadata)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]->getValueIntList(); + } + + public function getType(string $key): string { + return $this->metadata[$key]->getType(); + } + + public function set(string $key, string $value, bool $index = false): IFilesMetadata { + try { + if ($this->get($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value and index have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING); + $this->updated = true; + $this->metadata[$key] = $meta->setValueString($value)->setIndexed($index); + + return $this; + } + + public function setInt(string $key, int $value, bool $index = false): IFilesMetadata { + try { + if ($this->getInt($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_INT); + $this->metadata[$key] = $meta->setValueInt($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setFloat(string $key, float $value, bool $index = false): IFilesMetadata { + try { + if ($this->getFloat($key) === $value && $index === in_array($key, $this->getIndexes())) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_FLOAT); + $this->metadata[$key] = $meta->setValueFloat($value)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setBool(string $key, bool $value): IFilesMetadata { + try { + if ($this->getBool($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_BOOL); + $this->metadata[$key] = $meta->setValueBool($value); + $this->updated = true; + + return $this; + } + + public function setArray(string $key, array $value): IFilesMetadata { + try { + if ($this->getArray($key) === $value) { + return $this; // we ignore if value have not changed + } + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + // if value does not exist, or type has changed, we keep on the writing + } + + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_ARRAY); + $this->metadata[$key] = $meta->setValueArray($value); + $this->updated = true; + + return $this; + } + + + public function setStringList(string $key, array $values, bool $index = false): IFilesMetadata { + $meta = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $meta->setValueStringList($values)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function setIntList(string $key, array $values, bool $index = false): IFilesMetadata { + $valueWrapper = new MetadataValueWrapper(MetadataValueWrapper::TYPE_STRING_LIST); + $this->metadata[$key] = $valueWrapper->setValueIntList($values)->setIndexed($index); + $this->updated = true; + + return $this; + } + + public function unset(string $key): IFilesMetadata { + unset($this->metadata[$key]); + $this->updated = true; + + return $this; + } + + public function removeStartsWith(string $keyPrefix): IFilesMetadata { + if ($keyPrefix === '') { + return $this; + } + + foreach ($this->getKeys() as $key) { + if (str_starts_with($key, $keyPrefix)) { + $this->unset($key); + } + } + + return $this; + } + + /** + * @param string $key + * + * @return MetadataValueWrapper + * @throws FilesMetadataNotFoundException + */ + public function getValueWrapper(string $key): MetadataValueWrapper { + if (!$this->hasKey($key)) { + throw new FilesMetadataNotFoundException(); + } + + return $this->metadata[$key]; + } + + + public function jsonSerialize(): array { + return $this->metadata; + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataQuery.php b/lib/private/FilesMetadata/Model/MetadataQuery.php new file mode 100644 index 0000000000000..9c3271927cfd6 --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataQuery.php @@ -0,0 +1,80 @@ +queryBuilder->leftJoin( + $fileTableAlias, IndexRequestService::TABLE_METADATA_INDEX, $this->aliasIndex, + $this->queryBuilder->expr()->eq($this->aliasIndex . '.file_id', $fileTableAlias . '.' . $fileIdField) + ); + } + + public function retrieveMetadata(): void { + $this->queryBuilder->selectAlias($this->alias . '.json', 'meta_json'); + $this->queryBuilder->leftJoin( + $this->aliasIndex, MetadataRequestService::TABLE_METADATA, $this->alias, + $this->queryBuilder->expr()->eq($this->aliasIndex . '.file_id', $this->alias . '.file_id') + ); + } + + public function enforceMetadataKey(string $metaKey): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataKeyField(), + $this->queryBuilder->createNamedParameter($metaKey) + ) + ); + } + + public function enforceMetadataValue(string $value): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataKeyField(), + $this->queryBuilder->createNamedParameter($value) + ) + ); + } + + public function enforceMetadataValueInt(int $value): void { + $expr = $this->queryBuilder->expr(); + $this->queryBuilder->andWhere( + $expr->eq( + $this->getMetadataValueIntField(), + $this->queryBuilder->createNamedParameter($value, IQueryBuilder::PARAM_INT) + ) + ); + } + + public function getMetadataKeyField(): string { + return $this->aliasIndex . '.meta_key';// TODO: Implement getMetadataValueField() method. + } + + public function getMetadataValueField(): string { + return $this->aliasIndex . '.meta_value';// TODO: Implement getMetadataValueField() method. + } + + public function getMetadataValueIntField(): string { + return $this->aliasIndex . '.meta_value_int';// TODO: Implement getMetadataValueField() method. + } +} diff --git a/lib/private/FilesMetadata/Model/MetadataValueWrapper.php b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php new file mode 100644 index 0000000000000..132a5a8328d4a --- /dev/null +++ b/lib/private/FilesMetadata/Model/MetadataValueWrapper.php @@ -0,0 +1,295 @@ +type = $type; + } + + public function setType(string $type): self { + $this->type = $type; + return $this; + } + + public function getType(): string { + return $this->type; + } + + public function isType(string $type): bool { + return (strtolower($type) === strtolower($this->type)); + } + + /** + * confirm stored value exists and is typed as requested + * @param string $type + * + * @return $this + * @throws FilesMetadataTypeException + */ + public function confirmType(string $type): self { + if (!$this->isType($type)) { + throw new FilesMetadataTypeException('type is \'' . $this->getType() . '\', expecting \'' . $type . '\''); + } + + return $this; + } + + /** + * @param string $value + * + * @return $this + */ + public function setValueString(string $value): self { + if ($this->isType(self::TYPE_STRING)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param int $value + * + * @return $this + */ + public function setValueInt(int $value): self { + if ($this->isType(self::TYPE_INT)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param float $value + * + * @return $this + */ + public function setValueFloat(float $value): self { + if ($this->isType(self::TYPE_FLOAT)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param bool $value + * + * @return $this + */ + public function setValueBool(bool $value): self { + if ($this->isType(self::TYPE_BOOL)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param array $value + * + * @return $this + */ + public function setValueArray(array $value): self { + if ($this->isType(self::TYPE_ARRAY)) { + $this->value = $value; + } + + return $this; + } + + /** + * @param string[] $value + * + * @return $this + */ + public function setValueStringList(array $value): self { + if ($this->isType(self::TYPE_STRING_LIST)) { + // TODO confirm value is an array or string ? + $this->value = $value; + } + + return $this; + } + + /** + * @param int[] $value + * + * @return $this + */ + public function setValueIntList(array $value): self { + if ($this->isType(self::TYPE_INT_LIST)) { + // TODO confirm value is an array of int ? + $this->value = $value; + } + + return $this; + } + + + /** + * @return string + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueString(): string { + $this->confirmType(self::TYPE_STRING); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (string) $this->value; + } + + /** + * @return int + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueInt(): int { + $this->confirmType(self::TYPE_INT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (int) $this->value; + } + + /** + * @return float + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueFloat(): float { + $this->confirmType(self::TYPE_FLOAT); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (float) $this->value; + } + + /** + * @return bool + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueBool(): bool { + $this->confirmType(self::TYPE_BOOL); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (bool) $this->value; + } + + /** + * @return array + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueArray(): array { + $this->confirmType(self::TYPE_ARRAY); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + /** + * @return string[] + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueStringList(): array { + $this->confirmType(self::TYPE_STRING_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + /** + * @return array + * @throws FilesMetadataTypeException + * @throws FilesMetadataNotFoundException + */ + public function getValueIntList(): array { + $this->confirmType(self::TYPE_INT_LIST); + if (null === $this->value) { + throw new FilesMetadataNotFoundException('value is not set'); + } + + return (array) $this->value; + } + + + public function setIndexed(bool $indexed): self { + $this->indexed = $indexed; + + return $this; + } + + public function isIndexed(): bool { + return $this->indexed; + } + + public function import(array $data): self { + $this->value = $data['value'] ?? null; + $this->setType($data['type'] ?? ''); + $this->setIndexed($data['indexed'] ?? false); + + return $this; + } + + public function jsonSerialize(): array { + return [ + 'value' => $this->value, + 'type' => $this->getType(), + 'indexed' => $this->isIndexed() + ]; + } +} diff --git a/lib/private/FilesMetadata/Service/IndexRequestService.php b/lib/private/FilesMetadata/Service/IndexRequestService.php new file mode 100644 index 0000000000000..6804625de3235 --- /dev/null +++ b/lib/private/FilesMetadata/Service/IndexRequestService.php @@ -0,0 +1,131 @@ +getFileId(); + + /** + * might look harsh, but a lot simpler than comparing current indexed data, as we can expect + * conflict with a change of types. + * We assume that each time one random metadata were modified we can drop all index for this + * key and recreate them + */ + $this->dropIndex($fileId, $key); + + try { + match ($filesMetadata->getType($key)) { + MetadataValueWrapper::TYPE_STRING + => $this->insertIndexString($fileId, $key, $filesMetadata->get($key)), + MetadataValueWrapper::TYPE_INT + => $this->insertIndexInt($fileId, $key, $filesMetadata->getInt($key)), + MetadataValueWrapper::TYPE_STRING_LIST + => $this->insertIndexStringList($fileId, $key, $filesMetadata->getStringList($key)), + MetadataValueWrapper::TYPE_INT_LIST + => $this->insertIndexIntList($fileId, $key, $filesMetadata->getIntList($key)) + }; + } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { + $this->logger->warning('...'); + } + } + + + private function insertIndexString(int $fileId, string $key, string $value): void { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value', $qb->createNamedParameter($value)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + + public function insertIndexInt(int $fileId, string $key, int $value): void { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA_INDEX) + ->setValue('meta_key', $qb->createNamedParameter($key)) + ->setValue('meta_value_int', $qb->createNamedParameter($value, IQueryBuilder::PARAM_INT)) + ->setValue('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)); + $qb->executeStatement(); + } catch (Exception $e) { + $this->logger->warning('...'); + } + } + + /** + * @param int $fileId + * @param string $key + * @param string[] $values + * + * @return void + */ + public function insertIndexStringList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexString($fileId, $key, $value); + } + } + + /** + * @param int $fileId + * @param string $key + * @param int[] $values + * + * @return void + */ + public function insertIndexIntList(int $fileId, string $key, array $values): void { + foreach ($values as $value) { + $this->insertIndexInt($fileId, $key, $value); + } + } + + /** + * @param int $fileId + * @param string $key + * + * @return void + * @throws Exception + */ + public function dropIndex(int $fileId, string $key = ''): void { + $qb = $this->dbConnection->getQueryBuilder(); + $expr = $qb->expr(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($expr->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + + if ($key !== '') { + $qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key))); + } + + $qb->executeStatement(); + } +} diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php new file mode 100644 index 0000000000000..155789e987c34 --- /dev/null +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -0,0 +1,117 @@ +dbConnection->getQueryBuilder(); + $qb->insert(self::TABLE_METADATA) + ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)) + ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->setValue('sync_token', $qb->createNamedParameter($filesMetadata->getSyncToken())) + ->setValue('last_update', $qb->createFunction('NOW()')); + $qb->executeStatement(); + } + + /** + * @param int $fileId + * + * @return IFilesMetadata + * @throws FilesMetadataNotFoundException + */ + public function getMetadataFromFileId(int $fileId): IFilesMetadata { + try { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->select('json')->from(self::TABLE_METADATA); + $qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + $data = $result->fetch(); + $result->closeCursor(); + } catch (Exception $e) { + $this->logger->warning('exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId]); + throw new FilesMetadataNotFoundException(); + } + + if ($data === false) { + throw new FilesMetadataNotFoundException(); + } + + $metadata = new FilesMetadata($fileId); + try { + $metadata->import(json_decode($data['json'] ?? '[]', true, 512, JSON_THROW_ON_ERROR)); + } catch (JsonException $e) { + throw new FilesMetadataNotFoundException(); + } + + return $metadata; + } + + + /** + * @param int $fileId + * + * @return void + * @throws Exception + */ + public function dropMetadata(int $fileId): void { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + $qb->executeStatement(); + } + + private function removeDeprecatedMetadata(IFilesMetadata $filesMetadata): void { + // TODO delete aussi les index generate a partir d'une string[] + + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->notIn('file_id', $filesMetadata->getIndexes(), IQueryBuilder::PARAM_STR_ARRAY)); + $qb->executeStatement(); + } + + + /** + * @param IFilesMetadata $filesMetadata + * + * @return bool + * @throws Exception + */ + public function updateMetadata(IFilesMetadata $filesMetadata): int { + // TODO check sync_token on update + $qb = $this->dbConnection->getQueryBuilder(); + $qb->update(self::TABLE_METADATA) + ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize()))) + ->set('sync_token', $qb->createNamedParameter('abc')) + ->set('last_update', $qb->createFunction('NOW()')) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))); + + return $qb->executeStatement(); + } +} diff --git a/lib/private/Server.php b/lib/private/Server.php index d2a1d890ccd25..bfc1db7fffcf4 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -102,6 +102,7 @@ use OC\Files\Template\TemplateManager; use OC\Files\Type\Loader; use OC\Files\View; +use OC\FilesMetadata\FilesMetadataManager; use OC\FullTextSearch\FullTextSearchManager; use OC\Http\Client\ClientService; use OC\Http\Client\NegativeDnsCache; @@ -193,6 +194,7 @@ use OCP\Files\Mount\IMountManager; use OCP\Files\Storage\IStorageFactory; use OCP\Files\Template\ITemplateManager; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\FullTextSearch\IFullTextSearchManager; use OCP\GlobalScale\IConfig; use OCP\Group\ISubAdmin; @@ -1395,6 +1397,7 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(\OCP\Dashboard\IManager::class, \OC\Dashboard\Manager::class); $this->registerAlias(IFullTextSearchManager::class, FullTextSearchManager::class); + $this->registerAlias(IFilesMetadataManager::class, FilesMetadataManager::class); $this->registerAlias(ISubAdmin::class, SubAdmin::class); @@ -1466,6 +1469,8 @@ private function connectDispatcher(): void { $eventDispatcher->addServiceListener(PostLoginEvent::class, UserLoggedInListener::class); $eventDispatcher->addServiceListener(UserChangedEvent::class, UserChangedListener::class); $eventDispatcher->addServiceListener(BeforeUserDeletedEvent::class, BeforeUserDeletedListener::class); + + FilesMetadataManager::loadListeners($eventDispatcher); } /** diff --git a/lib/public/FilesMetadata/Event/MetadataEventBackgroundEvent.php b/lib/public/FilesMetadata/Event/MetadataEventBackgroundEvent.php new file mode 100644 index 0000000000000..4a43d71ef3626 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataEventBackgroundEvent.php @@ -0,0 +1,40 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\FilesMetadata\Event; + +use OC\FilesMetadata\Event\MetadataEventBase; +use OCP\Files\Node; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataEventBackgroundEvent extends MetadataEventBase { + public function __construct( + Node $node, + IFilesMetadata $metadata + ) { + parent::__construct($node, $metadata); + } +} diff --git a/lib/public/FilesMetadata/Event/MetadataEventLiveEvent.php b/lib/public/FilesMetadata/Event/MetadataEventLiveEvent.php new file mode 100644 index 0000000000000..5886449d68d00 --- /dev/null +++ b/lib/public/FilesMetadata/Event/MetadataEventLiveEvent.php @@ -0,0 +1,64 @@ + + * + * @author Maxence Lange + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\FilesMetadata\Event; + +use OC\FilesMetadata\Event\MetadataEventBase; +use OCP\Files\Node; +use OCP\FilesMetadata\Model\IFilesMetadata; + +class MetadataEventLiveEvent extends MetadataEventBase { + private bool $runAsBackgroundJob = false; + + public function __construct( + Node $node, + IFilesMetadata $metadata + ) { + parent::__construct($node, $metadata); + } + + /** + * If an app prefer to update metadata on a background job, instead of + * live process, just call this method. + * A new event will be generated on next cron tick. + * + * @return void + * @since 28.0.0 + */ + public function requestBackgroundJob(): void { + $this->runAsBackgroundJob = true; + } + + /** + * return true if any app that catch this event requested a re-run as background job + * + * @return bool + * @since 28.0.0 + */ + public function isRunAsBackgroundJobRequested(): bool { + return $this->runAsBackgroundJob; + } +} diff --git a/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php new file mode 100644 index 0000000000000..93685e22becf7 --- /dev/null +++ b/lib/public/FilesMetadata/Exceptions/FilesMetadataException.php @@ -0,0 +1,10 @@ +