IFilesMetadata

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

apps/files/lib/Command/Scan.php

@@ -37,17 +37,19 @@
 use OC\Core\Command\InterruptedException;
 use OC\DB\Connection;
 use OC\DB\ConnectionAdapter;
+use OC\FilesMetadata\FilesMetadataManager;
+use OC\ForbiddenException;
+use OC\Metadata\MetadataManager;
+use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\Events\FileCacheUpdated;
 use OCP\Files\Events\NodeAddedToCache;
 use OCP\Files\Events\NodeRemovedFromCache;
 use OCP\Files\File;
-use OC\ForbiddenException;
-use OC\Metadata\MetadataManager;
-use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\IRootFolder;
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\NotFoundException;
 use OCP\Files\StorageNotAvailableException;
+use OCP\FilesMetadata\IFilesMetadataManager;
 use OCP\IUserManager;
 use Psr\Log\LoggerInterface;
 use Symfony\Component\Console\Helper\Table;
@@ -69,6 +71,7 @@ public function __construct(
 		private IUserManager $userManager,
 		private IRootFolder $rootFolder,
 		private MetadataManager $metadataManager,
+		private FilesMetadataManager $filesMetadataManager,
 		private IEventDispatcher $eventDispatcher,
 		private LoggerInterface $logger,
 	) {
@@ -140,6 +143,11 @@ protected function scanFiles(string $user, string $path, bool $scanMetadata, Out
 				if ($node instanceof File) {
 					$this->metadataManager->generateMetadata($node, false);
 				}
+
+				$this->filesMetadataManager->refreshMetadata(
+					$node,
+					IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND
+				);
 			}
 		});
 

apps/files_trashbin/lib/Trash/TrashItem.php

@@ -190,4 +190,8 @@ public function getUploadTime(): int {
 	public function getParentId(): int {
 		return $this->fileInfo->getParentId();
 	}
+
+	public function getMetadata(): array {
+		return $this->fileInfo->getMetadata();
+	}
 }

core/Command/FilesMetadata/Get.php

@@ -0,0 +1,105 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Command\FilesMetadata;
+
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use OCP\FilesMetadata\IFilesMetadataManager;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Get extends Command {
+	public function __construct(
+		private IRootFolder $rootFolder,
+		private IFilesMetadataManager $filesMetadataManager,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this->setName('metadata:get')
+			 ->setDescription('get stored metadata about a file, by its id')
+			 ->addArgument(
+				 'fileId',
+				 InputArgument::REQUIRED,
+				 'id of the file document'
+			 )
+			 ->addArgument(
+				 'userId',
+				 InputArgument::OPTIONAL,
+				 'file owner'
+			 )
+			 ->addOption(
+				 'as-array',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'display metadata as a simple key=>value array'
+			 )
+			 ->addOption(
+				'refresh',
+				'',
+				InputOption::VALUE_NONE,
+				'refresh metadata'
+			)
+			 ->addOption(
+				 'reset',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'refresh metadata from scratch'
+			 );
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$fileId = (int)$input->getArgument('fileId');
+		if ($input->getOption('refresh')) {
+			$node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getById($fileId);
+			$file = $node[0];
+			if (null === $file) {
+				throw new NotFoundException();
+			}
+
+			$metadata = $this->filesMetadataManager->refreshMetadata(
+				$file,
+				IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND,
+				$input->getOption('reset')
+			);
+		} else {
+			$metadata = $this->filesMetadataManager->getMetadata($fileId);
+		}
+
+		if ($input->getOption('as-array')) {
+			$output->writeln(json_encode($metadata->asArray(), JSON_PRETTY_PRINT));
+		} else {
+			$output->writeln(json_encode($metadata, JSON_PRETTY_PRINT));
+		}
+
+		return 0;
+	}
+}

core/Migrations/Version28000Date20231004103301.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version28000Date20231004103301 extends SimpleMigrationStep {
+
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('files_metadata')) {
+			$table = $schema->createTable('files_metadata');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15,]);
+			$table->addColumn('json', Types::TEXT);
+			$table->addColumn('sync_token', Types::STRING, ['length' => 15]);
+			$table->addColumn('last_update', Types::DATETIME);
+
+			$table->setPrimaryKey(['id']);
+			$table->addUniqueIndex(['file_id'], 'files_meta_fileid');
+		}
+
+		if (!$schema->hasTable('files_metadata_index')) {
+			$table = $schema->createTable('files_metadata_index');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15]);
+			$table->addColumn('meta_key', Types::STRING, ['notnull' => false, 'length' => 31]);
+			$table->addColumn('meta_value', Types::STRING, ['notnull' => false, 'length' => 63]);
+			$table->addColumn('meta_value_int', Types::BIGINT, ['notnull' => false, 'length' => 11]);
+//			$table->addColumn('meta_value_float', Types::FLOAT, ['notnull' => false, 'length' => 11,5]);
+
+			$table->setPrimaryKey(['id']);
+			$table->addIndex(['file_id', 'meta_key', 'meta_value'], 'f_meta_index');
+			$table->addIndex(['file_id', 'meta_key', 'meta_value_int'], 'f_meta_index_i');
+//			$table->addIndex(['file_id', 'meta_key', 'meta_value_float'], 'f_meta_index_f');
+		}
+
+		return $schema;
+	}
+}

core/register_command.php

@@ -214,6 +214,8 @@
 	$application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager()));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class));
+
+	$application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class));
 } else {
 	$application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class));
 }

lib/private/Files/Cache/Cache.php

@@ -169,6 +169,9 @@ public function get($file) {
 		$data = $result->fetch();
 		$result->closeCursor();
 
+		// @Louis: use asArray()
+//		$data['metadata'] = $metadataQuery?->extractMetadata($data)?->asArray() ?? [];
+
 		//merge partial data
 		if (!$data && is_string($file) && isset($this->partial[$file])) {
 			return $this->partial[$file];

lib/private/Files/Cache/CacheQueryBuilder.php

@@ -35,7 +35,7 @@
  * Query builder with commonly used helpers for filecache queries
  */
 class CacheQueryBuilder extends QueryBuilder {
-	private $alias = null;
+	private ?string $alias = null;
 
 	public function __construct(IDBConnection $connection, SystemConfig $systemConfig, LoggerInterface $logger) {
 		parent::__construct($connection, $systemConfig, $logger);

lib/private/Files/Cache/QuerySearchHelper.php

@@ -37,41 +37,24 @@
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\Search\ISearchBinaryOperator;
 use OCP\Files\Search\ISearchQuery;
+use OCP\FilesMetadata\IFilesMetadataManager;
+use OCP\FilesMetadata\Model\IMetadataQuery;
 use OCP\IDBConnection;
 use OCP\IGroupManager;
 use OCP\IUser;
 use Psr\Log\LoggerInterface;
 
 class QuerySearchHelper {
-	/** @var IMimeTypeLoader */
-	private $mimetypeLoader;
-	/** @var IDBConnection */
-	private $connection;
-	/** @var SystemConfig */
-	private $systemConfig;
-	private LoggerInterface $logger;
-	/** @var SearchBuilder */
-	private $searchBuilder;
-	/** @var QueryOptimizer */
-	private $queryOptimizer;
-	private IGroupManager $groupManager;
-
 	public function __construct(
-		IMimeTypeLoader $mimetypeLoader,
-		IDBConnection $connection,
-		SystemConfig $systemConfig,
-		LoggerInterface $logger,
-		SearchBuilder $searchBuilder,
-		QueryOptimizer $queryOptimizer,
-		IGroupManager $groupManager,
+		private IMimeTypeLoader $mimetypeLoader,
+		private IDBConnection $connection,
+		private SystemConfig $systemConfig,
+		private LoggerInterface $logger,
+		private SearchBuilder $searchBuilder,
+		private QueryOptimizer $queryOptimizer,
+		private IGroupManager $groupManager,
+		private IFilesMetadataManager $filesMetadataManager,
 	) {
-		$this->mimetypeLoader = $mimetypeLoader;
-		$this->connection = $connection;
-		$this->systemConfig = $systemConfig;
-		$this->logger = $logger;
-		$this->searchBuilder = $searchBuilder;
-		$this->queryOptimizer = $queryOptimizer;
-		$this->groupManager = $groupManager;
 	}
 
 	protected function getQueryBuilder() {
@@ -144,6 +127,36 @@ protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user):
 			));
 	}
 
+
+	/**
+	 * left join metadata and its indexes to the filecache table
+	 *
+	 * @param CacheQueryBuilder $query
+	 * @param ISearchQuery $searchQuery
+	 *
+	 * @return IMetadataQuery|null
+	 */
+	protected function equipQueryForMetadata(CacheQueryBuilder $query, ISearchQuery $searchQuery): ?IMetadataQuery {
+		$metadataQuery = $this->filesMetadataManager->getMetadataQuery($query, 'file', 'fileid');
+		$metadataQuery->retrieveMetadata();
+
+		$order = $searchQuery->getOrder();
+		if ($order) {
+			foreach ($order as $orderField) {
+				$metadataQuery->joinIndex($orderField->getField());
+				$query->orderBy($metadataQuery->getMetadataValueIntField(), $orderField->getDirection());
+			}
+		}
+
+		// TODO: add filter on metadatakey / metadatavalue
+		// is it possible to get information from the webdav request ?
+		// $expr = $query->expr();
+		// $query->andWhere($expr->eq($metadataQuery->getMetadataKeyField(), $query->createNamedParameter('my_key')));
+		// $query->andWhere($expr->eq($metadataQuery->getMetadataValueField(), $query->createNamedParameter('my_value')));
+
+		return $metadataQuery;
+	}
+
 	/**
 	 * Perform a file system search in multiple caches
 	 *
@@ -175,19 +188,22 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array
 		$query = $builder->selectFileCache('file', false);
 
 		$requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation());
+
 		if (in_array('systemtag', $requestedFields)) {
 			$this->equipQueryForSystemTags($query, $this->requireUser($searchQuery));
 		}
 		if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) {
 			$this->equipQueryForDavTags($query, $this->requireUser($searchQuery));
 		}
 
+		$metadataQuery = $this->equipQueryForMetadata($query, $searchQuery);
 		$this->applySearchConstraints($query, $searchQuery, $caches);
 
 		$result = $query->execute();
 		$files = $result->fetchAll();
 
-		$rawEntries = array_map(function (array $data) {
+		$rawEntries = array_map(function (array $data) use ($metadataQuery) {
+			$data['metadata'] = $metadataQuery?->extractMetadata($data)?->asArray() ?? [];
 			return Cache::cacheEntryFromData($data, $this->mimetypeLoader);
 		}, $files);
 

lib/private/Files/FileInfo.php

@@ -32,6 +32,7 @@
  */
 namespace OC\Files;
 
+use OC\FilesMetadata\Model\FilesMetadata;
 use OCA\Files_Sharing\ISharedStorage;
 use OCP\Files\Cache\ICacheEntry;
 use OCP\Files\IHomeStorage;
@@ -416,4 +417,8 @@ public function getUploadTime(): int {
 	public function getParentId(): int {
 		return $this->data['parent'] ?? -1;
 	}
+
+	public function getMetadata(): array {
+		return $this->data['metadata'] ?? [];
+	}
 }