From d3a313f192c090d026bac1fd1a8aed718d54c634 Mon Sep 17 00:00:00 2001 From: Louis Chemineau Date: Wed, 8 Nov 2023 12:35:01 +0100 Subject: [PATCH] Support getting and setting metadata in DAV requests Signed-off-by: Louis Chemineau --- apps/dav/lib/Connector/Sabre/Directory.php | 4 - apps/dav/lib/Connector/Sabre/File.php | 16 -- apps/dav/lib/Connector/Sabre/FilesPlugin.php | 127 ++++++------- apps/dav/lib/Files/FileSearchBackend.php | 23 --- .../unit/Connector/Sabre/FilesPluginTest.php | 3 + apps/files/lib/Command/Scan.php | 7 - apps/files_trashbin/lib/Trashbin.php | 4 +- core/Application.php | 17 -- lib/composer/composer/autoload_classmap.php | 8 - lib/composer/composer/autoload_static.php | 8 - lib/private/Files/Cache/Cache.php | 11 +- lib/private/Files/Cache/CacheQueryBuilder.php | 15 +- lib/private/Files/Cache/QuerySearchHelper.php | 20 +- .../FilesMetadata/FilesMetadataManager.php | 15 +- .../FilesMetadata/Listener/MetadataUpdate.php | 2 +- .../FilesMetadata/Model/FilesMetadata.php | 8 +- .../Service/IndexRequestService.php | 2 +- lib/private/Metadata/Capabilities.php | 42 ----- lib/private/Metadata/FileEventListener.php | 108 ----------- lib/private/Metadata/FileMetadata.php | 51 ----- lib/private/Metadata/FileMetadataMapper.php | 177 ------------------ lib/private/Metadata/IMetadataManager.php | 35 ---- lib/private/Metadata/IMetadataProvider.php | 41 ---- lib/private/Metadata/MetadataManager.php | 95 ---------- .../Metadata/Provider/ExifProvider.php | 139 -------------- lib/private/Server.php | 8 - .../FilesMetadata/IFilesMetadataManager.php | 3 +- .../FilesMetadata/Model/IFilesMetadata.php | 4 +- tests/lib/Metadata/FileMetadataMapperTest.php | 87 --------- 29 files changed, 109 insertions(+), 971 deletions(-) delete mode 100644 lib/private/Metadata/Capabilities.php delete mode 100644 lib/private/Metadata/FileEventListener.php delete mode 100644 lib/private/Metadata/FileMetadata.php delete mode 100644 lib/private/Metadata/FileMetadataMapper.php delete mode 100644 lib/private/Metadata/IMetadataManager.php delete mode 100644 lib/private/Metadata/IMetadataProvider.php delete mode 100644 lib/private/Metadata/MetadataManager.php delete mode 100644 lib/private/Metadata/Provider/ExifProvider.php delete mode 100644 tests/lib/Metadata/FileMetadataMapperTest.php diff --git a/apps/dav/lib/Connector/Sabre/Directory.php b/apps/dav/lib/Connector/Sabre/Directory.php index 20202d8368956..ab321a363a2c7 100644 --- a/apps/dav/lib/Connector/Sabre/Directory.php +++ b/apps/dav/lib/Connector/Sabre/Directory.php @@ -34,7 +34,6 @@ use OC\Files\Mount\MoveableMount; use OC\Files\View; -use OC\Metadata\FileMetadata; use OCA\DAV\AppInfo\Application; use OCA\DAV\Connector\Sabre\Exception\FileLocked; use OCA\DAV\Connector\Sabre\Exception\Forbidden; @@ -70,9 +69,6 @@ class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICol private ?array $quotaInfo = null; private ?CachingTree $tree = null; - /** @var array> */ - private array $metadata = []; - /** * Sets up the node, expects a full path name */ diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index 8ce8c843a6628..f188490fd938f 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -43,7 +43,6 @@ use OC\Files\Filesystem; use OC\Files\Stream\HashWrapper; use OC\Files\View; -use OC\Metadata\FileMetadata; use OCA\DAV\AppInfo\Application; use OCA\DAV\Connector\Sabre\Exception\BadGateway; use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge; @@ -81,9 +80,6 @@ class File extends Node implements IFile { protected IRequest $request; protected IL10N $l10n; - /** @var array */ - private array $metadata = []; - /** * Sets up the node, expects a full path name * @@ -796,16 +792,4 @@ public function hash(string $type) { public function getNode(): \OCP\Files\File { return $this->node; } - - public function getMetadata(string $group): FileMetadata { - return $this->metadata[$group]; - } - - public function setMetadata(string $group, FileMetadata $metadata): void { - $this->metadata[$group] = $metadata; - } - - public function hasMetadata(string $group) { - return array_key_exists($group, $this->metadata); - } } diff --git a/apps/dav/lib/Connector/Sabre/FilesPlugin.php b/apps/dav/lib/Connector/Sabre/FilesPlugin.php index 9319327d094c9..cd188872019cc 100644 --- a/apps/dav/lib/Connector/Sabre/FilesPlugin.php +++ b/apps/dav/lib/Connector/Sabre/FilesPlugin.php @@ -35,19 +35,20 @@ namespace OCA\DAV\Connector\Sabre; use OC\AppFramework\Http\Request; -use OC\Metadata\IMetadataManager; +use OC\FilesMetadata\Model\MetadataValueWrapper; use OCP\Constants; use OCP\Files\ForbiddenException; use OCP\Files\StorageNotAvailableException; +use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\Model\IMetadataValueWrapper; use OCP\IConfig; use OCP\IPreview; use OCP\IRequest; use OCP\IUserSession; -use Psr\Log\LoggerInterface; use Sabre\DAV\Exception\Forbidden; use Sabre\DAV\Exception\NotFound; use Sabre\DAV\IFile; -use Sabre\DAV\INode; use Sabre\DAV\PropFind; use Sabre\DAV\PropPatch; use Sabre\DAV\Server; @@ -86,17 +87,6 @@ class FilesPlugin extends ServerPlugin { public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count'; public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count'; public const FILE_METADATA_PREFIX = '{http://nextcloud.org/ns}metadata-'; - public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size'; - public const FILE_METADATA_GPS = '{http://nextcloud.org/ns}file-metadata-gps'; - - public const ALL_METADATA_PROPS = [ - self::FILE_METADATA_SIZE => 'size', - self::FILE_METADATA_GPS => 'gps', - ]; - public const METADATA_MIMETYPES = [ - 'size' => 'image', - 'gps' => 'image', - ]; /** Reference to main server object */ private ?Server $server = null; @@ -434,31 +424,6 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node) $propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) { return $node->getFileInfo()->getUploadTime(); }); - - if ($this->config->getSystemValueBool('enable_file_metadata', true)) { - foreach (self::ALL_METADATA_PROPS as $prop => $meta) { - $propFind->handle($prop, function () use ($node, $meta) { - if ($node->getFileInfo()->getMimePart() !== self::METADATA_MIMETYPES[$meta]) { - return []; - } - - if ($node->hasMetadata($meta)) { - $metadata = $node->getMetadata($meta); - } else { - // This code path should not be called since we try to preload - // the metadata when loading the folder or the search results - // in one go - $metadataManager = \OC::$server->get(IMetadataManager::class); - $metadata = $metadataManager->fetchMetadataFor($meta, [$node->getId()])[$node->getId()]; - - // TODO would be nice to display this in the profiler... - \OC::$server->get(LoggerInterface::class)->debug('Inefficient fetching of metadata'); - } - - return $metadata->getValue(); - }); - } - } } if ($node instanceof Directory) { @@ -472,39 +437,6 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node) $requestProperties = $propFind->getRequestedProperties(); - $requestedMetaData = []; - foreach ($requestProperties as $requestProperty) { - if (isset(self::ALL_METADATA_PROPS[$requestProperty])) { - $requestedMetaData[] = self::ALL_METADATA_PROPS[$requestProperty]; - } - } - if ( - $this->config->getSystemValueBool('enable_file_metadata', true) && - $propFind->getDepth() === 1 && - $requestedMetaData - ) { - $children = $node->getChildren(); - // Preloading of the metadata - - /** @var IMetaDataManager $metadataManager */ - $metadataManager = \OC::$server->get(IMetadataManager::class); - - foreach ($requestedMetaData as $requestedMeta) { - $relevantMimeType = self::METADATA_MIMETYPES[$requestedMeta]; - $childrenForMeta = array_filter($children, function (INode $child) use ($relevantMimeType) { - return $child instanceof File && $child->getFileInfo()->getMimePart() === $relevantMimeType; - }); - $fileIds = array_map(function (File $child) { - return $child->getFileInfo()->getId(); - }, $childrenForMeta); - $preloadedMetadata = $metadataManager->fetchMetadataFor($requestedMeta, $fileIds); - - foreach ($childrenForMeta as $child) { - $child->setMetadata($requestedMeta, $preloadedMetadata[$child->getFileInfo()->getId()]); - } - } - } - if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true) || in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) { $nbFiles = 0; @@ -590,6 +522,57 @@ public function handleUpdateProperties($path, PropPatch $propPatch) { $node->setCreationTime((int) $time); return true; }); + + + /** @var IFilesMetadataManager */ + $filesMetadataManager = \OCP\Server::get(IFilesMetadataManager::class); + $knownMetadata = $filesMetadataManager->getKnownMetadata(); + + foreach ($propPatch->getRemainingMutations() as $mutation) { + if (!str_starts_with($mutation, self::FILE_METADATA_PREFIX)) { + continue; + } + + $propPatch->handle($mutation, function (mixed $value) use ($knownMetadata, $node, $mutation, $filesMetadataManager): bool { + $metadata = $filesMetadataManager->getMetadata((int)$node->getFileId(), true); + $metadataKey = substr($mutation, strlen(self::FILE_METADATA_PREFIX)); + + // If the metadata is unknown, it defaults to string. + try { + $type = $knownMetadata->getType($metadataKey); + } catch (FilesMetadataNotFoundException) { + $type = IMetadataValueWrapper::TYPE_STRING; + } + + switch ($type) { + case IMetadataValueWrapper::TYPE_STRING: + $metadata->setString($metadataKey, $value, $knownMetadata->isIndex($metadataKey)); + break; + case IMetadataValueWrapper::TYPE_INT: + $metadata->setInt($metadataKey, $value, $knownMetadata->isIndex($metadataKey)); + break; + case IMetadataValueWrapper::TYPE_FLOAT: + $metadata->setFloat($metadataKey, $value); + break; + case IMetadataValueWrapper::TYPE_BOOL: + $metadata->setBool($metadataKey, $value, $knownMetadata->isIndex($metadataKey)); + break; + case IMetadataValueWrapper::TYPE_ARRAY: + $metadata->setArray($metadataKey, $value); + break; + case IMetadataValueWrapper::TYPE_STRING_LIST: + $metadata->setStringList($metadataKey, $value, $knownMetadata->isIndex($metadataKey)); + break; + case IMetadataValueWrapper::TYPE_INT_LIST: + $metadata->setIntList($metadataKey, $value, $knownMetadata->isIndex($metadataKey)); + break; + } + + $filesMetadataManager->saveMetadata($metadata); + return true; + }); + } + /** * Disable modification of the displayname property for files and * folders via PROPPATCH. See PROPFIND for more information. diff --git a/apps/dav/lib/Files/FileSearchBackend.php b/apps/dav/lib/Files/FileSearchBackend.php index 658ad34ec32c8..b158fde857c2d 100644 --- a/apps/dav/lib/Files/FileSearchBackend.php +++ b/apps/dav/lib/Files/FileSearchBackend.php @@ -31,7 +31,6 @@ use OC\Files\Search\SearchOrder; use OC\Files\Search\SearchQuery; use OC\Files\View; -use OC\Metadata\IMetadataManager; use OCA\DAV\Connector\Sabre\CachingTree; use OCA\DAV\Connector\Sabre\Directory; use OCA\DAV\Connector\Sabre\FilesPlugin; @@ -115,7 +114,6 @@ public function getPropertyDefinitionsForScope(string $href, ?string $path): arr new SearchPropertyDefinition(FilesPlugin::OWNER_DISPLAY_NAME_PROPERTYNAME, true, false, false), new SearchPropertyDefinition(FilesPlugin::DATA_FINGERPRINT_PROPERTYNAME, true, false, false), new SearchPropertyDefinition(FilesPlugin::HAS_PREVIEW_PROPERTYNAME, true, false, false, SearchPropertyDefinition::DATATYPE_BOOLEAN), - new SearchPropertyDefinition(FilesPlugin::FILE_METADATA_SIZE, true, false, false, SearchPropertyDefinition::DATATYPE_STRING), new SearchPropertyDefinition(FilesPlugin::FILEID_PROPERTYNAME, true, false, false, SearchPropertyDefinition::DATATYPE_NONNEGATIVE_INTEGER), ]; @@ -152,27 +150,6 @@ private function getPropertyDefinitionsForMetadata(): array { * @param string[] $requestProperties */ public function preloadPropertyFor(array $nodes, array $requestProperties): void { - if (in_array(FilesPlugin::FILE_METADATA_SIZE, $requestProperties, true)) { - // Preloading of the metadata - $fileIds = []; - foreach ($nodes as $node) { - /** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */ - if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) { - /** @var \OCA\DAV\Connector\Sabre\File $node */ - $fileIds[] = $node->getFileInfo()->getId(); - } - } - /** @var IMetaDataManager $metadataManager */ - $metadataManager = \OC::$server->get(IMetadataManager::class); - $preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds); - foreach ($nodes as $node) { - /** @var \OCP\Files\Node|\OCA\DAV\Connector\Sabre\Node $node */ - if (str_starts_with($node->getFileInfo()->getMimeType(), 'image/')) { - /** @var \OCA\DAV\Connector\Sabre\File $node */ - $node->setMetadata('size', $preloadedMetadata[$node->getFileInfo()->getId()]); - } - } - } } /** diff --git a/apps/dav/tests/unit/Connector/Sabre/FilesPluginTest.php b/apps/dav/tests/unit/Connector/Sabre/FilesPluginTest.php index 5e638bbcd89f8..32573a68f7339 100644 --- a/apps/dav/tests/unit/Connector/Sabre/FilesPluginTest.php +++ b/apps/dav/tests/unit/Connector/Sabre/FilesPluginTest.php @@ -57,6 +57,8 @@ * This file is licensed under the Affero General Public License version 3 or * later. * See the COPYING-README file. + * + * @group DB */ class FilesPluginTest extends TestCase { public const GETETAG_PROPERTYNAME = FilesPlugin::GETETAG_PROPERTYNAME; @@ -424,6 +426,7 @@ public function testUpdateProps(): void { self::CREATIONDATE_PROPERTYNAME => $testCreationDate, ]); + $this->plugin->handleUpdateProperties( '/dummypath', $propPatch diff --git a/apps/files/lib/Command/Scan.php b/apps/files/lib/Command/Scan.php index b1fc25bfe9b3c..7cdaf75e9bc58 100644 --- a/apps/files/lib/Command/Scan.php +++ b/apps/files/lib/Command/Scan.php @@ -40,12 +40,10 @@ use OC\DB\ConnectionAdapter; use OC\FilesMetadata\FilesMetadataManager; use OC\ForbiddenException; -use OC\Metadata\MetadataManager; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Events\FileCacheUpdated; use OCP\Files\Events\NodeAddedToCache; use OCP\Files\Events\NodeRemovedFromCache; -use OCP\Files\File; use OCP\Files\IRootFolder; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; @@ -71,7 +69,6 @@ class Scan extends Base { public function __construct( private IUserManager $userManager, private IRootFolder $rootFolder, - private MetadataManager $metadataManager, private FilesMetadataManager $filesMetadataManager, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, @@ -141,10 +138,6 @@ protected function scanFiles(string $user, string $path, bool $scanMetadata, Out $this->abortIfInterrupted(); if ($scanMetadata) { $node = $this->rootFolder->get($path); - if ($node instanceof File) { - $this->metadataManager->generateMetadata($node, false); - } - $this->filesMetadataManager->refreshMetadata( $node, IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND diff --git a/apps/files_trashbin/lib/Trashbin.php b/apps/files_trashbin/lib/Trashbin.php index 90f74e51e8167..94ad77ac1c553 100644 --- a/apps/files_trashbin/lib/Trashbin.php +++ b/apps/files_trashbin/lib/Trashbin.php @@ -59,6 +59,7 @@ use OCP\Files\Folder; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IConfig; use OCP\Lock\ILockingProvider; use OCP\Lock\LockedException; @@ -993,7 +994,8 @@ private static function getVersionsFromTrash($filename, $timestamp, string $user $query = new CacheQueryBuilder( \OC::$server->getDatabaseConnection(), \OC::$server->getSystemConfig(), - \OC::$server->get(LoggerInterface::class) + \OC::$server->get(LoggerInterface::class), + \OC::$server->get(IFilesMetadataManager::class), ); $normalizedParentPath = ltrim(Filesystem::normalizePath(dirname('files_trashbin/versions/'. $filename)), '/'); $parentId = $cache->getId($normalizedParentPath); diff --git a/core/Application.php b/core/Application.php index ca9b6ce2d8c1b..2ad8b9f2a3014 100644 --- a/core/Application.php +++ b/core/Application.php @@ -44,7 +44,6 @@ use OC\Authentication\Notifications\Notifier as AuthenticationNotifier; use OC\Core\Listener\BeforeTemplateRenderedListener; use OC\Core\Notification\CoreNotifier; -use OC\Metadata\FileEventListener; use OC\TagManager; use OCP\AppFramework\App; use OCP\AppFramework\Http\Events\BeforeLoginTemplateRenderedEvent; @@ -54,13 +53,9 @@ use OCP\DB\Events\AddMissingPrimaryKeyEvent; use OCP\DB\Types; use OCP\EventDispatcher\IEventDispatcher; -use OCP\Files\Events\Node\NodeDeletedEvent; -use OCP\Files\Events\Node\NodeWrittenEvent; -use OCP\Files\Events\NodeRemovedFromCache; use OCP\User\Events\BeforeUserDeletedEvent; use OCP\User\Events\UserDeletedEvent; use OCP\Util; -use OCP\IConfig; /** * Class Application @@ -331,18 +326,6 @@ public function __construct() { $eventDispatcher->addServiceListener(UserDeletedEvent::class, UserDeletedFilesCleanupListener::class); $eventDispatcher->addServiceListener(UserDeletedEvent::class, UserDeletedWebAuthnCleanupListener::class); - // Metadata - /** @var IConfig $config */ - $config = $container->get(IConfig::class); - if ($config->getSystemValueBool('enable_file_metadata', true)) { - /** @psalm-suppress InvalidArgument */ - $eventDispatcher->addServiceListener(NodeDeletedEvent::class, FileEventListener::class); - /** @psalm-suppress InvalidArgument */ - $eventDispatcher->addServiceListener(NodeRemovedFromCache::class, FileEventListener::class); - /** @psalm-suppress InvalidArgument */ - $eventDispatcher->addServiceListener(NodeWrittenEvent::class, FileEventListener::class); - } - // Tags $eventDispatcher->addServiceListener(UserDeletedEvent::class, TagManager::class); } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index aaba8312c44db..1a55209c28635 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1496,14 +1496,6 @@ 'OC\\Memcache\\Redis' => $baseDir . '/lib/private/Memcache/Redis.php', 'OC\\Memcache\\WithLocalCache' => $baseDir . '/lib/private/Memcache/WithLocalCache.php', 'OC\\MemoryInfo' => $baseDir . '/lib/private/MemoryInfo.php', - 'OC\\Metadata\\Capabilities' => $baseDir . '/lib/private/Metadata/Capabilities.php', - 'OC\\Metadata\\FileEventListener' => $baseDir . '/lib/private/Metadata/FileEventListener.php', - 'OC\\Metadata\\FileMetadata' => $baseDir . '/lib/private/Metadata/FileMetadata.php', - 'OC\\Metadata\\FileMetadataMapper' => $baseDir . '/lib/private/Metadata/FileMetadataMapper.php', - 'OC\\Metadata\\IMetadataManager' => $baseDir . '/lib/private/Metadata/IMetadataManager.php', - 'OC\\Metadata\\IMetadataProvider' => $baseDir . '/lib/private/Metadata/IMetadataProvider.php', - 'OC\\Metadata\\MetadataManager' => $baseDir . '/lib/private/Metadata/MetadataManager.php', - 'OC\\Metadata\\Provider\\ExifProvider' => $baseDir . '/lib/private/Metadata/Provider/ExifProvider.php', 'OC\\Migration\\BackgroundRepair' => $baseDir . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => $baseDir . '/lib/private/Migration/ConsoleOutput.php', 'OC\\Migration\\SimpleOutput' => $baseDir . '/lib/private/Migration/SimpleOutput.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 6c341797b79fc..bebf5be91bdd2 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1529,14 +1529,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Memcache\\Redis' => __DIR__ . '/../../..' . '/lib/private/Memcache/Redis.php', 'OC\\Memcache\\WithLocalCache' => __DIR__ . '/../../..' . '/lib/private/Memcache/WithLocalCache.php', 'OC\\MemoryInfo' => __DIR__ . '/../../..' . '/lib/private/MemoryInfo.php', - 'OC\\Metadata\\Capabilities' => __DIR__ . '/../../..' . '/lib/private/Metadata/Capabilities.php', - 'OC\\Metadata\\FileEventListener' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileEventListener.php', - 'OC\\Metadata\\FileMetadata' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileMetadata.php', - 'OC\\Metadata\\FileMetadataMapper' => __DIR__ . '/../../..' . '/lib/private/Metadata/FileMetadataMapper.php', - 'OC\\Metadata\\IMetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataManager.php', - 'OC\\Metadata\\IMetadataProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/IMetadataProvider.php', - 'OC\\Metadata\\MetadataManager' => __DIR__ . '/../../..' . '/lib/private/Metadata/MetadataManager.php', - 'OC\\Metadata\\Provider\\ExifProvider' => __DIR__ . '/../../..' . '/lib/private/Metadata/Provider/ExifProvider.php', 'OC\\Migration\\BackgroundRepair' => __DIR__ . '/../../..' . '/lib/private/Migration/BackgroundRepair.php', 'OC\\Migration\\ConsoleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/ConsoleOutput.php', 'OC\\Migration\\SimpleOutput' => __DIR__ . '/../../..' . '/lib/private/Migration/SimpleOutput.php', diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 67d01bb699907..240f02b3fbae7 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -59,6 +59,7 @@ use OCP\Files\Search\ISearchOperator; use OCP\Files\Search\ISearchQuery; use OCP\Files\Storage\IStorage; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; use OCP\Util; use Psr\Log\LoggerInterface; @@ -132,7 +133,8 @@ protected function getQueryBuilder() { return new CacheQueryBuilder( $this->connection, \OC::$server->getSystemConfig(), - \OC::$server->get(LoggerInterface::class) + \OC::$server->get(LoggerInterface::class), + \OC::$server->get(IFilesMetadataManager::class), ); } @@ -154,6 +156,7 @@ public function getNumericStorageId() { public function get($file) { $query = $this->getQueryBuilder(); $query->selectFileCache(); + $metadataQuery = $query->selectMetadata(); if (is_string($file) || $file == '') { // normalize file @@ -175,6 +178,7 @@ public function get($file) { } elseif (!$data) { return $data; } else { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); } } @@ -239,11 +243,14 @@ public function getFolderContentsById($fileId) { ->whereParent($fileId) ->orderBy('name', 'ASC'); + $metadataQuery = $query->selectMetadata(); + $result = $query->execute(); $files = $result->fetchAll(); $result->closeCursor(); - return array_map(function (array $data) { + return array_map(function (array $data) use ($metadataQuery) { + $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray(); return self::cacheEntryFromData($data, $this->mimetypeLoader); }, $files); } diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php index f799d6aa72e73..27f66e63e7bf3 100644 --- a/lib/private/Files/Cache/CacheQueryBuilder.php +++ b/lib/private/Files/Cache/CacheQueryBuilder.php @@ -29,6 +29,8 @@ use OC\DB\QueryBuilder\QueryBuilder; use OC\SystemConfig; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\FilesMetadata\Model\IMetadataQuery; use OCP\IDBConnection; use Psr\Log\LoggerInterface; @@ -38,7 +40,12 @@ class CacheQueryBuilder extends QueryBuilder { private ?string $alias = null; - public function __construct(IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger) { + public function __construct( + IDBConnection $connection, + SystemConfig $systemConfig, + LoggerInterface $logger, + private IFilesMetadataManager $filesMetadataManager, + ) { parent::__construct($connection, $systemConfig, $logger); } @@ -127,4 +134,10 @@ public function whereParentInParameter(string $parameter) { return $this; } + + public function selectMetadata(): IMetadataQuery { + $metadataQuery = $this->filesMetadataManager->getMetadataQuery($this, $this->alias, 'fileid'); + $metadataQuery->retrieveMetadata(); + return $metadataQuery; + } } diff --git a/lib/private/Files/Cache/QuerySearchHelper.php b/lib/private/Files/Cache/QuerySearchHelper.php index ca54133a24319..f8e5d1608f742 100644 --- a/lib/private/Files/Cache/QuerySearchHelper.php +++ b/lib/private/Files/Cache/QuerySearchHelper.php @@ -62,7 +62,8 @@ protected function getQueryBuilder() { return new CacheQueryBuilder( $this->connection, $this->systemConfig, - $this->logger + $this->logger, + $this->filesMetadataManager, ); } @@ -133,20 +134,6 @@ protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user): )); } - - /** - * left join metadata and its indexes to the filecache table - * - * @param CacheQueryBuilder $query - * - * @return IMetadataQuery - */ - protected function equipQueryForMetadata(CacheQueryBuilder $query): IMetadataQuery { - $metadataQuery = $this->filesMetadataManager->getMetadataQuery($query, 'file', 'fileid'); - $metadataQuery->retrieveMetadata(); - return $metadataQuery; - } - /** * Perform a file system search in multiple caches * @@ -186,7 +173,8 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array $this->equipQueryForDavTags($query, $this->requireUser($searchQuery)); } - $metadataQuery = $this->equipQueryForMetadata($query); + $metadataQuery = $query->selectMetadata(); + $this->applySearchConstraints($query, $searchQuery, $caches, $metadataQuery); $result = $query->execute(); diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php index 54310f934d72a..b4c91c3836a50 100644 --- a/lib/private/FilesMetadata/FilesMetadataManager.php +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -38,7 +38,6 @@ use OCP\DB\Exception as DBException; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; -use OCP\Files\Events\Node\NodeCreatedEvent; use OCP\Files\Events\Node\NodeDeletedEvent; use OCP\Files\Events\Node\NodeWrittenEvent; use OCP\Files\InvalidPathException; @@ -124,14 +123,23 @@ public function refreshMetadata( /** * @param int $fileId file id + * @param boolean $generate Generate if metadata does not exists * * @inheritDoc * @return IFilesMetadata * @throws FilesMetadataNotFoundException if not found * @since 28.0.0 */ - public function getMetadata(int $fileId): IFilesMetadata { - return $this->metadataRequestService->getMetadataFromFileId($fileId); + public function getMetadata(int $fileId, bool $generate = false): IFilesMetadata { + try { + return $this->metadataRequestService->getMetadataFromFileId($fileId); + } catch (FilesMetadataNotFoundException $ex) { + if ($generate) { + return new FilesMetadata($fileId); + } + + throw $ex; + } } /** @@ -274,7 +282,6 @@ public function initMetadata(string $key, string $type, bool $indexed): void { * @param IEventDispatcher $eventDispatcher */ public static function loadListeners(IEventDispatcher $eventDispatcher): void { - $eventDispatcher->addServiceListener(NodeCreatedEvent::class, MetadataUpdate::class); $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); $eventDispatcher->addServiceListener(NodeDeletedEvent::class, MetadataDelete::class); } diff --git a/lib/private/FilesMetadata/Listener/MetadataUpdate.php b/lib/private/FilesMetadata/Listener/MetadataUpdate.php index 395a852e9e390..9848f079882ea 100644 --- a/lib/private/FilesMetadata/Listener/MetadataUpdate.php +++ b/lib/private/FilesMetadata/Listener/MetadataUpdate.php @@ -51,7 +51,7 @@ public function __construct( * @param Event $event */ public function handle(Event $event): void { - if (!($event instanceof NodeCreatedEvent) && !($event instanceof NodeWrittenEvent)) { + if (!($event instanceof NodeWrittenEvent)) { return; } diff --git a/lib/private/FilesMetadata/Model/FilesMetadata.php b/lib/private/FilesMetadata/Model/FilesMetadata.php index a94c7a9b6ff18..b10de55579c75 100644 --- a/lib/private/FilesMetadata/Model/FilesMetadata.php +++ b/lib/private/FilesMetadata/Model/FilesMetadata.php @@ -133,7 +133,7 @@ public function isIndex(string $key): bool { * @throws FilesMetadataTypeException * @since 28.0.0 */ - public function get(string $key): string { + public function getString(string $key): string { if (!array_key_exists($key, $this->metadata)) { throw new FilesMetadataNotFoundException(); } @@ -276,10 +276,10 @@ public function getType(string $key): string { * @throws FilesMetadataKeyFormatException * @since 28.0.0 */ - public function set(string $key, string $value, bool $index = false): IFilesMetadata { + public function setString(string $key, string $value, bool $index = false): IFilesMetadata { $this->confirmKeyFormat($key); try { - if ($this->get($key) === $value && $index === $this->isIndex($key)) { + if ($this->getString($key) === $value && $index === $this->isIndex($key)) { return $this; // we ignore if value and index have not changed } } catch (FilesMetadataNotFoundException|FilesMetadataTypeException $e) { @@ -506,7 +506,7 @@ private function confirmKeyFormat(string $key): void { return; } - throw new FilesMetadataKeyFormatException('key can only contains alphanumerical characters, and dash (-)'); + throw new FilesMetadataKeyFormatException('key can only contains alphanumerical characters, and dash (-, _)'); } /** diff --git a/lib/private/FilesMetadata/Service/IndexRequestService.php b/lib/private/FilesMetadata/Service/IndexRequestService.php index 6530dabd1c3d9..2a23e2c9a6726 100644 --- a/lib/private/FilesMetadata/Service/IndexRequestService.php +++ b/lib/private/FilesMetadata/Service/IndexRequestService.php @@ -73,7 +73,7 @@ public function updateIndex(IFilesMetadata $filesMetadata, string $key): void { try { $this->dropIndex($fileId, $key); match ($metadataType) { - IMetadataValueWrapper::TYPE_STRING => $this->insertIndexString($fileId, $key, $filesMetadata->get($key)), + IMetadataValueWrapper::TYPE_STRING => $this->insertIndexString($fileId, $key, $filesMetadata->getString($key)), IMetadataValueWrapper::TYPE_INT => $this->insertIndexInt($fileId, $key, $filesMetadata->getInt($key)), IMetadataValueWrapper::TYPE_BOOL => $this->insertIndexBool($fileId, $key, $filesMetadata->getBool($key)), IMetadataValueWrapper::TYPE_STRING_LIST => $this->insertIndexStringList($fileId, $key, $filesMetadata->getStringList($key)), diff --git a/lib/private/Metadata/Capabilities.php b/lib/private/Metadata/Capabilities.php deleted file mode 100644 index d8b0b82377e43..0000000000000 --- a/lib/private/Metadata/Capabilities.php +++ /dev/null @@ -1,42 +0,0 @@ - - * @license AGPL-3.0-or-later - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Metadata; - -use OCP\Capabilities\IPublicCapability; -use OCP\IConfig; - -class Capabilities implements IPublicCapability { - public function __construct( - private IMetadataManager $manager, - private IConfig $config, - ) { - } - - public function getCapabilities(): array { - if ($this->config->getSystemValueBool('enable_file_metadata', true)) { - return ['metadataAvailable' => $this->manager->getCapabilities()]; - } - - return []; - } -} diff --git a/lib/private/Metadata/FileEventListener.php b/lib/private/Metadata/FileEventListener.php deleted file mode 100644 index 5a1882ac0e42c..0000000000000 --- a/lib/private/Metadata/FileEventListener.php +++ /dev/null @@ -1,108 +0,0 @@ - - * @license AGPL-3.0-or-later - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Metadata; - -use OC\Files\Filesystem; -use OCP\EventDispatcher\Event; -use OCP\EventDispatcher\IEventListener; -use OCP\Files\Events\Node\NodeDeletedEvent; -use OCP\Files\Events\Node\NodeWrittenEvent; -use OCP\Files\Events\NodeRemovedFromCache; -use OCP\Files\File; -use OCP\Files\Node; -use OCP\Files\NotFoundException; -use OCP\Files\FileInfo; -use Psr\Log\LoggerInterface; - -/** - * @template-implements IEventListener - * @template-implements IEventListener - * @template-implements IEventListener - */ -class FileEventListener implements IEventListener { - public function __construct( - private IMetadataManager $manager, - private LoggerInterface $logger, - ) { - } - - private function shouldExtractMetadata(Node $node): bool { - try { - if ($node->getMimetype() === 'httpd/unix-directory') { - return false; - } - } catch (NotFoundException $e) { - return false; - } - if ($node->getSize(false) <= 0) { - return false; - } - - $path = $node->getPath(); - return $this->isCorrectPath($path); - } - - private function isCorrectPath(string $path): bool { - // TODO make this more dynamic, we have the same issue in other places - return !str_starts_with($path, 'appdata_') && !str_starts_with($path, 'files_versions/') && !str_starts_with($path, 'files_trashbin/'); - } - - public function handle(Event $event): void { - if ($event instanceof NodeRemovedFromCache) { - if (!$this->isCorrectPath($event->getPath())) { - // Don't listen to paths for which we don't extract metadata - return; - } - $view = Filesystem::getView(); - if (!$view) { - // Should not happen since a scan in the user folder should setup - // the file system. - $e = new \Exception(); // don't trigger, just get backtrace - $this->logger->error('Detecting deletion of a file with possible metadata but file system setup is not setup', [ - 'exception' => $e, - 'app' => 'metadata' - ]); - return; - } - $info = $view->getFileInfo($event->getPath()); - if ($info && $info->getType() === FileInfo::TYPE_FILE) { - $this->manager->clearMetadata($info->getId()); - } - } - - if ($event instanceof NodeDeletedEvent) { - $node = $event->getNode(); - if ($this->shouldExtractMetadata($node)) { - /** @var File $node */ - $this->manager->clearMetadata($event->getNode()->getId()); - } - } - - if ($event instanceof NodeWrittenEvent) { - $node = $event->getNode(); - if ($this->shouldExtractMetadata($node)) { - /** @var File $node */ - $this->manager->generateMetadata($event->getNode(), false); - } - } - } -} diff --git a/lib/private/Metadata/FileMetadata.php b/lib/private/Metadata/FileMetadata.php deleted file mode 100644 index a9808a86998e3..0000000000000 --- a/lib/private/Metadata/FileMetadata.php +++ /dev/null @@ -1,51 +0,0 @@ - - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Metadata; - -use OCP\AppFramework\Db\Entity; -use OCP\DB\Types; - -/** - * @method string getGroupName() - * @method void setGroupName(string $groupName) - * @method string getValue() - * @method void setValue(string $value) - * @see \OC\Core\Migrations\Version240000Date20220404230027 - */ -class FileMetadata extends Entity { - protected ?string $groupName = null; - protected ?string $value = null; - - public function __construct() { - $this->addType('groupName', 'string'); - $this->addType('value', Types::STRING); - } - - public function getDecodedValue(): array { - return json_decode($this->getValue(), true) ?? []; - } - - public function setArrayAsValue(array $value): void { - $this->setValue(json_encode($value, JSON_THROW_ON_ERROR)); - } -} diff --git a/lib/private/Metadata/FileMetadataMapper.php b/lib/private/Metadata/FileMetadataMapper.php deleted file mode 100644 index 003ab13126e88..0000000000000 --- a/lib/private/Metadata/FileMetadataMapper.php +++ /dev/null @@ -1,177 +0,0 @@ - - * @copyright Copyright 2022 Louis Chmn - * @license AGPL-3.0-or-later - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Metadata; - -use OCP\AppFramework\Db\DoesNotExistException; -use OCP\AppFramework\Db\MultipleObjectsReturnedException; -use OCP\AppFramework\Db\QBMapper; -use OCP\AppFramework\Db\Entity; -use OCP\DB\Exception; -use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\IDBConnection; - -/** - * @template-extends QBMapper - */ -class FileMetadataMapper extends QBMapper { - public function __construct(IDBConnection $db) { - parent::__construct($db, 'file_metadata', FileMetadata::class); - } - - /** - * @return FileMetadata[] - * @throws Exception - */ - public function findForFile(int $fileId): array { - $qb = $this->db->getQueryBuilder(); - $qb->select('*') - ->from($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); - - return $this->findEntities($qb); - } - - /** - * @throws DoesNotExistException - * @throws MultipleObjectsReturnedException - * @throws Exception - */ - public function findForGroupForFile(int $fileId, string $groupName): FileMetadata { - $qb = $this->db->getQueryBuilder(); - $qb->select('*') - ->from($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, IQueryBuilder::PARAM_STR))); - - return $this->findEntity($qb); - } - - /** - * @return array - * @throws Exception - */ - public function findForGroupForFiles(array $fileIds, string $groupName): array { - $qb = $this->db->getQueryBuilder(); - $qb->select('*') - ->from($this->getTableName()) - ->where($qb->expr()->in('id', $qb->createParameter('fileIds'))) - ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, IQueryBuilder::PARAM_STR))); - - $metadata = []; - foreach (array_chunk($fileIds, 1000) as $fileIdsChunk) { - $qb->setParameter('fileIds', $fileIdsChunk, IQueryBuilder::PARAM_INT_ARRAY); - /** @var FileMetadata[] $rawEntities */ - $rawEntities = $this->findEntities($qb); - foreach ($rawEntities as $entity) { - $metadata[$entity->getId()] = $entity; - } - } - - foreach ($fileIds as $id) { - if (isset($metadata[$id])) { - continue; - } - $empty = new FileMetadata(); - $empty->setValue(''); - $empty->setGroupName($groupName); - $empty->setId($id); - $metadata[$id] = $empty; - } - return $metadata; - } - - public function clear(int $fileId): void { - $qb = $this->db->getQueryBuilder(); - $qb->delete($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); - - $qb->executeStatement(); - } - - /** - * Updates an entry in the db from an entity - * - * @param FileMetadata $entity the entity that should be created - * @return FileMetadata the saved entity with the set id - * @throws Exception - * @throws \InvalidArgumentException if entity has no id - */ - public function update(Entity $entity): FileMetadata { - if (!($entity instanceof FileMetadata)) { - throw new \Exception("Entity should be a FileMetadata entity"); - } - - // entity needs an id - $id = $entity->getId(); - if ($id === null) { - throw new \InvalidArgumentException('Entity which should be updated has no id'); - } - - // entity needs an group_name - $groupName = $entity->getGroupName(); - if ($groupName === null) { - throw new \InvalidArgumentException('Entity which should be updated has no group_name'); - } - - $idType = $this->getParameterTypeForProperty($entity, 'id'); - $groupNameType = $this->getParameterTypeForProperty($entity, 'groupName'); - $value = $entity->getValue(); - $valueType = $this->getParameterTypeForProperty($entity, 'value'); - - $qb = $this->db->getQueryBuilder(); - - $qb->update($this->tableName) - ->set('value', $qb->createNamedParameter($value, $valueType)) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($id, $idType))) - ->andWhere($qb->expr()->eq('group_name', $qb->createNamedParameter($groupName, $groupNameType))) - ->executeStatement(); - - return $entity; - } - - /** - * Override the insertOrUpdate as we could be in a transaction in which case we can not afford on error. - * - * @param FileMetadata $entity the entity that should be created/updated - * @return FileMetadata the saved entity with the (new) id - * @throws Exception - * @throws \InvalidArgumentException if entity has no id - */ - public function insertOrUpdate(Entity $entity): FileMetadata { - try { - $existingEntity = $this->findForGroupForFile($entity->getId(), $entity->getGroupName()); - } catch (\Throwable) { - $existingEntity = null; - } - - if ($existingEntity !== null) { - if ($entity->getValue() !== $existingEntity->getValue()) { - return $this->update($entity); - } else { - return $existingEntity; - } - } else { - return parent::insertOrUpdate($entity); - } - } -} diff --git a/lib/private/Metadata/IMetadataManager.php b/lib/private/Metadata/IMetadataManager.php deleted file mode 100644 index fa0bcc2280166..0000000000000 --- a/lib/private/Metadata/IMetadataManager.php +++ /dev/null @@ -1,35 +0,0 @@ - $className - */ - public function registerProvider(string $className): void; - - /** - * Generate the metadata for one file - */ - public function generateMetadata(File $file, bool $checkExisting = false): void; - - /** - * Clear the metadata for one file - */ - public function clearMetadata(int $fileId): void; - - /** @return array */ - public function fetchMetadataFor(string $group, array $fileIds): array; - - /** - * Get the capabilities as an array of mimetype regex to the type provided - */ - public function getCapabilities(): array; -} diff --git a/lib/private/Metadata/IMetadataProvider.php b/lib/private/Metadata/IMetadataProvider.php deleted file mode 100644 index 7cbe102a53887..0000000000000 --- a/lib/private/Metadata/IMetadataProvider.php +++ /dev/null @@ -1,41 +0,0 @@ - An array containing all the metadata fetched. - */ - public function execute(File $file): array; -} diff --git a/lib/private/Metadata/MetadataManager.php b/lib/private/Metadata/MetadataManager.php deleted file mode 100644 index ef9f9200df7d8..0000000000000 --- a/lib/private/Metadata/MetadataManager.php +++ /dev/null @@ -1,95 +0,0 @@ - - * @license AGPL-3.0-or-later - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Metadata; - -use OC\Metadata\Provider\ExifProvider; -use OCP\Files\File; - -class MetadataManager implements IMetadataManager { - /** @var array */ - private array $providers = []; - private array $providerClasses = []; - - public function __construct( - private FileMetadataMapper $fileMetadataMapper, - ) { - // TODO move to another place, where? - $this->registerProvider(ExifProvider::class); - } - - /** - * @param class-string $className - */ - public function registerProvider(string $className):void { - if (in_array($className, $this->providerClasses)) { - return; - } - - if (call_user_func([$className, 'isAvailable'])) { - $this->providers[call_user_func([$className, 'getMimetypesSupported'])] = \OC::$server->get($className); - } - } - - public function generateMetadata(File $file, bool $checkExisting = false): void { - $existingMetadataGroups = []; - - if ($checkExisting) { - $existingMetadata = $this->fileMetadataMapper->findForFile($file->getId()); - foreach ($existingMetadata as $metadata) { - $existingMetadataGroups[] = $metadata->getGroupName(); - } - } - - foreach ($this->providers as $supportedMimetype => $provider) { - if (preg_match($supportedMimetype, $file->getMimeType())) { - if (count(array_diff($provider::groupsProvided(), $existingMetadataGroups)) > 0) { - $metaDataGroup = $provider->execute($file); - foreach ($metaDataGroup as $group => $metadata) { - $this->fileMetadataMapper->insertOrUpdate($metadata); - } - } - } - } - } - - public function clearMetadata(int $fileId): void { - $this->fileMetadataMapper->clear($fileId); - } - - /** - * @return array - */ - public function fetchMetadataFor(string $group, array $fileIds): array { - return $this->fileMetadataMapper->findForGroupForFiles($fileIds, $group); - } - - public function getCapabilities(): array { - $capabilities = []; - foreach ($this->providers as $supportedMimetype => $provider) { - foreach ($provider::groupsProvided() as $group) { - if (isset($capabilities[$group])) { - $capabilities[$group][] = $supportedMimetype; - } - $capabilities[$group] = [$supportedMimetype]; - } - } - return $capabilities; - } -} diff --git a/lib/private/Metadata/Provider/ExifProvider.php b/lib/private/Metadata/Provider/ExifProvider.php deleted file mode 100644 index 26edc6e01a04a..0000000000000 --- a/lib/private/Metadata/Provider/ExifProvider.php +++ /dev/null @@ -1,139 +0,0 @@ - - * @copyright Copyright 2022 Louis Chmn - * @license AGPL-3.0-or-later - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace OC\Metadata\Provider; - -use OC\Metadata\FileMetadata; -use OC\Metadata\IMetadataProvider; -use OCP\Files\File; -use Psr\Log\LoggerInterface; - -class ExifProvider implements IMetadataProvider { - public function __construct( - private LoggerInterface $logger, - ) { - } - - public static function groupsProvided(): array { - return ['size', 'gps']; - } - - public static function isAvailable(): bool { - return extension_loaded('exif'); - } - - /** @return array{'gps'?: FileMetadata, 'size'?: FileMetadata} */ - public function execute(File $file): array { - $exifData = []; - $fileDescriptor = $file->fopen('rb'); - - if ($fileDescriptor === false) { - return []; - } - - $data = null; - try { - // Needed to make reading exif data reliable. - // This is to trigger this condition: https://github.com/php/php-src/blob/d64aa6f646a7b5e58359dc79479860164239580a/main/streams/streams.c#L710 - // But I don't understand why 1 as a special meaning. - // Revert right after reading the exif data. - $oldBufferSize = stream_set_chunk_size($fileDescriptor, 1); - $data = @exif_read_data($fileDescriptor, 'ANY_TAG', true); - stream_set_chunk_size($fileDescriptor, $oldBufferSize); - } catch (\Exception $ex) { - $this->logger->info("Couldn't extract metadata for ".$file->getId(), ['exception' => $ex]); - } - - $size = new FileMetadata(); - $size->setGroupName('size'); - $size->setId($file->getId()); - $size->setArrayAsValue([]); - - if (!$data) { - $sizeResult = getimagesizefromstring($file->getContent()); - if ($sizeResult !== false) { - $size->setArrayAsValue([ - 'width' => $sizeResult[0], - 'height' => $sizeResult[1], - ]); - - $exifData['size'] = $size; - } - } elseif (array_key_exists('COMPUTED', $data)) { - if (array_key_exists('Width', $data['COMPUTED']) && array_key_exists('Height', $data['COMPUTED'])) { - $size->setArrayAsValue([ - 'width' => $data['COMPUTED']['Width'], - 'height' => $data['COMPUTED']['Height'], - ]); - - $exifData['size'] = $size; - } - } - - if ($data && array_key_exists('GPS', $data) - && array_key_exists('GPSLatitude', $data['GPS']) && array_key_exists('GPSLatitudeRef', $data['GPS']) - && array_key_exists('GPSLongitude', $data['GPS']) && array_key_exists('GPSLongitudeRef', $data['GPS']) - ) { - $gps = new FileMetadata(); - $gps->setGroupName('gps'); - $gps->setId($file->getId()); - $gps->setArrayAsValue([ - 'latitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLatitude'], $data['GPS']['GPSLatitudeRef']), - 'longitude' => $this->gpsDegreesToDecimal($data['GPS']['GPSLongitude'], $data['GPS']['GPSLongitudeRef']), - ]); - - $exifData['gps'] = $gps; - } - - return $exifData; - } - - public static function getMimetypesSupported(): string { - return '/image\/(png|jpeg|heif|webp|tiff)/'; - } - - /** - * @param array|string $coordinates - */ - private static function gpsDegreesToDecimal($coordinates, ?string $hemisphere): float { - if (is_string($coordinates)) { - $coordinates = array_map("trim", explode(",", $coordinates)); - } - - if (count($coordinates) !== 3) { - throw new \Exception('Invalid coordinate format: ' . json_encode($coordinates)); - } - - [$degrees, $minutes, $seconds] = array_map(function (string $rawDegree) { - $parts = explode('/', $rawDegree); - - if ($parts[1] === '0') { - return 0; - } - - return floatval($parts[0]) / floatval($parts[1] ?? 1); - }, $coordinates); - - $sign = ($hemisphere === 'W' || $hemisphere === 'S') ? -1 : 1; - return $sign * ($degrees + $minutes / 60 + $seconds / 3600); - } -} diff --git a/lib/private/Server.php b/lib/private/Server.php index 31e4a9f79e2d5..80f9ba8bdd1b1 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -121,9 +121,6 @@ use OC\Mail\Mailer; use OC\Memcache\ArrayCache; use OC\Memcache\Factory; -use OC\Metadata\Capabilities as MetadataCapabilities; -use OC\Metadata\IMetadataManager; -use OC\Metadata\MetadataManager; use OC\Notification\Manager; use OC\OCM\Model\OCMProvider; use OC\OCM\OCMDiscoveryService; @@ -1136,9 +1133,6 @@ public function __construct($webRoot, \OC\Config $config) { $manager->registerCapability(function () use ($c) { return $c->get(\OC\Security\Bruteforce\Capabilities::class); }); - $manager->registerCapability(function () use ($c) { - return $c->get(MetadataCapabilities::class); - }); return $manager; }); /** @deprecated 19.0.0 */ @@ -1415,8 +1409,6 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(IBroker::class, Broker::class); - $this->registerAlias(IMetadataManager::class, MetadataManager::class); - $this->registerAlias(\OCP\Files\AppData\IAppDataFactory::class, \OC\Files\AppData\Factory::class); $this->registerAlias(IBinaryFinder::class, BinaryFinder::class); diff --git a/lib/public/FilesMetadata/IFilesMetadataManager.php b/lib/public/FilesMetadata/IFilesMetadataManager.php index 1cd0fcb412587..61494cac37143 100644 --- a/lib/public/FilesMetadata/IFilesMetadataManager.php +++ b/lib/public/FilesMetadata/IFilesMetadataManager.php @@ -69,12 +69,13 @@ public function refreshMetadata( * returns metadata from a file id * * @param int $fileId file id + * @param boolean $generate Generate if metadata does not exist * * @return IFilesMetadata * @throws FilesMetadataNotFoundException if not found * @since 28.0.0 */ - public function getMetadata(int $fileId): IFilesMetadata; + public function getMetadata(int $fileId, bool $generate = false): IFilesMetadata; /** * save metadata to database and refresh indexes. diff --git a/lib/public/FilesMetadata/Model/IFilesMetadata.php b/lib/public/FilesMetadata/Model/IFilesMetadata.php index ca1cc898f74d4..e5b6ff1bce03d 100644 --- a/lib/public/FilesMetadata/Model/IFilesMetadata.php +++ b/lib/public/FilesMetadata/Model/IFilesMetadata.php @@ -120,7 +120,7 @@ public function isIndex(string $key): bool; * @throws FilesMetadataTypeException * @since 28.0.0 */ - public function get(string $key): string; + public function getString(string $key): string; /** * returns int value for a metadata key @@ -222,7 +222,7 @@ public function getType(string $key): string; * @return self * @since 28.0.0 */ - public function set(string $key, string $value, bool $index = false): self; + public function setString(string $key, string $value, bool $index = false): self; /** * set a metadata key/value pair for int value diff --git a/tests/lib/Metadata/FileMetadataMapperTest.php b/tests/lib/Metadata/FileMetadataMapperTest.php deleted file mode 100644 index 4f7708ab9a92a..0000000000000 --- a/tests/lib/Metadata/FileMetadataMapperTest.php +++ /dev/null @@ -1,87 +0,0 @@ - - * @license AGPL-3.0-or-later - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -namespace Test\Metadata; - -use OC\Metadata\FileMetadataMapper; -use OC\Metadata\FileMetadata; -use PHPUnit\Framework\MockObject\MockObject; - -/** - * @group DB - * @package Test\DB\QueryBuilder - */ -class FileMetadataMapperTest extends \Test\TestCase { - /** @var IDBConnection */ - protected $connection; - - /** @var SystemConfig|MockObject */ - protected $config; - - /** @var FileMetadataMapper|MockObject */ - protected $mapper; - - protected function setUp(): void { - parent::setUp(); - - $this->connection = \OC::$server->getDatabaseConnection(); - $this->mapper = new FileMetadataMapper($this->connection); - } - - public function testFindForGroupForFiles() { - $file1 = new FileMetadata(); - $file1->setId(1); - $file1->setGroupName('size'); - $file1->setArrayAsValue([]); - - $file2 = new FileMetadata(); - $file2->setId(2); - $file2->setGroupName('size'); - $file2->setArrayAsValue(['width' => 293, 'height' => 23]); - - // not added, it's the default - $file3 = new FileMetadata(); - $file3->setId(3); - $file3->setGroupName('size'); - $file3->setArrayAsValue([]); - - $file4 = new FileMetadata(); - $file4->setId(4); - $file4->setGroupName('size'); - $file4->setArrayAsValue(['complex' => ["yes", "maybe" => 34.0]]); - - $this->mapper->insert($file1); - $this->mapper->insert($file2); - $this->mapper->insert($file4); - - $files = $this->mapper->findForGroupForFiles([1, 2, 3, 4], 'size'); - - $this->assertEquals($files[1]->getValue(), $file1->getValue()); - $this->assertEquals($files[2]->getValue(), $file2->getValue()); - $this->assertEquals($files[3]->getDecodedValue(), $file3->getDecodedValue()); - $this->assertEquals($files[4]->getValue(), $file4->getValue()); - - $this->mapper->clear(1); - $this->mapper->clear(2); - $this->mapper->clear(4); - } -}