FilesMetadata

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

apps/files/lib/Command/Scan.php

@@ -37,6 +37,7 @@
 use OC\Core\Command\InterruptedException;
 use OC\DB\Connection;
 use OC\DB\ConnectionAdapter;
+use OC\FilesMetadata\FilesMetadataManager;
 use OCP\Files\Events\FileCacheUpdated;
 use OCP\Files\Events\NodeAddedToCache;
 use OCP\Files\Events\NodeRemovedFromCache;
@@ -69,6 +70,7 @@ public function __construct(
 		private IUserManager $userManager,
 		private IRootFolder $rootFolder,
 		private MetadataManager $metadataManager,
+		private FilesMetadataManager $filesMetadataManager,
 		private IEventDispatcher $eventDispatcher,
 		private LoggerInterface $logger,
 	) {
@@ -140,6 +142,8 @@ protected function scanFiles(string $user, string $path, bool $scanMetadata, Out
 				if ($node instanceof File) {
 					$this->metadataManager->generateMetadata($node, false);
 				}
+
+				$this->filesMetadataManager->refreshMetadata($node);
 			}
 		});
 

core/Command/FilesMetadata/Get.php

@@ -0,0 +1,98 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Command\FilesMetadata;
+
+use OC\DB\ConnectionAdapter;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\IRootFolder;
+use OCP\FilesMetadata\IFilesMetadataManager;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Get extends Command {
+	public function __construct(
+		private IRootFolder $rootFolder,
+		private IFilesMetadataManager $filesMetadataManager,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this->setName('metadata:get')
+			 ->setDescription('get stored metadata about a file, by its id')
+			 ->addArgument(
+				 'fileId',
+				 InputArgument::REQUIRED,
+				 'id of the file document'
+			 )
+			 ->addArgument(
+				 'userId',
+				 InputArgument::OPTIONAL,
+				 'file owner'
+			 )
+			 ->addOption(
+				 'refresh',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'refresh metadata'
+			 )
+			 ->addOption(
+				 'reset',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'refresh metadata from scratch'
+			 )
+			 ->addOption(
+				 'background',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'emulate background jobs when refreshing metadata'
+			 );
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$fileId = (int)$input->getArgument('fileId');
+		if ($input->getOption('refresh')) {
+			$node = $this->rootFolder->getUserFolder($input->getArgument('userId'))->getById($fileId);
+			$file = $node[0];
+			$metadata = $this->filesMetadataManager->refreshMetadata(
+				$file,
+				$input->getOption('background'),
+				$input->getOption('reset')
+			);
+		} else {
+			$metadata = $this->filesMetadataManager->getMetadata($fileId);
+		}
+
+		$output->writeln(json_encode($metadata, JSON_PRETTY_PRINT));
+
+		return 0;
+	}
+}

core/Migrations/Version28000Date20231004103301.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version28000Date20231004103301 extends SimpleMigrationStep {
+
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('files_metadata')) {
+			$table = $schema->createTable('files_metadata');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15,]);
+			$table->addColumn('json', Types::TEXT);
+			$table->addColumn('sync_token', Types::STRING, ['length' => 15]);
+			$table->addColumn('last_update', Types::DATETIME);
+
+			$table->setPrimaryKey(['id']);
+			$table->addUniqueIndex(['file_id'], 'files_meta_fileid');
+		}
+
+		if (!$schema->hasTable('files_metadata_index')) {
+			$table = $schema->createTable('files_metadata_index');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, ['notnull' => false, 'length' => 15]);
+			$table->addColumn('meta_key', Types::STRING, ['notnull' => false, 'length' => 31]);
+			$table->addColumn('meta_value', Types::STRING, ['notnull' => false, 'length' => 63]);
+			$table->addColumn('meta_value_int', Types::BIGINT, ['notnull' => false, 'length' => 11]);
+//			$table->addColumn('meta_value_float', Types::FLOAT, ['notnull' => false, 'length' => 11,5]);
+
+			$table->setPrimaryKey(['id']);
+			$table->addIndex(['file_id', 'meta_key', 'meta_value'], 'f_meta_index');
+			$table->addIndex(['file_id', 'meta_key', 'meta_value_int'], 'f_meta_index_i');
+//			$table->addIndex(['file_id', 'meta_key', 'meta_value_float'], 'f_meta_index_f');
+		}
+
+		return $schema;
+	}
+}

core/register_command.php

@@ -214,6 +214,8 @@
 	$application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager()));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class));
+
+	$application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class));
 } else {
 	$application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class));
 }

lib/private/Files/Cache/QuerySearchHelper.php

@@ -25,9 +25,11 @@
  */
 namespace OC\Files\Cache;
 
+use OC\DB\ConnectionAdapter;
 use OC\Files\Cache\Wrapper\CacheJail;
 use OC\Files\Search\QueryOptimizer\QueryOptimizer;
 use OC\Files\Search\SearchBinaryOperator;
+use OC\FilesMetadata\Model\MetadataQuery;
 use OC\SystemConfig;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\Cache\ICache;
@@ -37,41 +39,24 @@
 use OCP\Files\Mount\IMountPoint;
 use OCP\Files\Search\ISearchBinaryOperator;
 use OCP\Files\Search\ISearchQuery;
+use OCP\FilesMetadata\IFilesMetadataManager;
 use OCP\IDBConnection;
 use OCP\IGroupManager;
 use OCP\IUser;
 use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Output\OutputInterface;
 
 class QuerySearchHelper {
-	/** @var IMimeTypeLoader */
-	private $mimetypeLoader;
-	/** @var IDBConnection */
-	private $connection;
-	/** @var SystemConfig */
-	private $systemConfig;
-	private LoggerInterface $logger;
-	/** @var SearchBuilder */
-	private $searchBuilder;
-	/** @var QueryOptimizer */
-	private $queryOptimizer;
-	private IGroupManager $groupManager;
-
 	public function __construct(
-		IMimeTypeLoader $mimetypeLoader,
-		IDBConnection $connection,
-		SystemConfig $systemConfig,
-		LoggerInterface $logger,
-		SearchBuilder $searchBuilder,
-		QueryOptimizer $queryOptimizer,
-		IGroupManager $groupManager,
+		private IMimeTypeLoader $mimetypeLoader,
+		private IDBConnection $connection,
+		private SystemConfig $systemConfig,
+		private LoggerInterface $logger,
+		private SearchBuilder $searchBuilder,
+		private QueryOptimizer $queryOptimizer,
+		private IGroupManager $groupManager,
+		private IFilesMetadataManager $filesMetadataManager,
 	) {
-		$this->mimetypeLoader = $mimetypeLoader;
-		$this->connection = $connection;
-		$this->systemConfig = $systemConfig;
-		$this->logger = $logger;
-		$this->searchBuilder = $searchBuilder;
-		$this->queryOptimizer = $queryOptimizer;
-		$this->groupManager = $groupManager;
 	}
 
 	protected function getQueryBuilder() {
@@ -144,6 +129,23 @@ protected function equipQueryForDavTags(CacheQueryBuilder $query, IUser $user):
 			));
 	}
 
+
+	protected function equipQueryForMetadata(CacheQueryBuilder $query, ISearchQuery $searchQuery): ?MetadataQuery {
+		// TODO: use $searchQuery to improve the query
+
+		// init the thing
+		$metadataQuery = $this->filesMetadataManager->getMetadataQuery($query, 'fc', 'fileid');
+
+		// get metadata aside the files
+		$metadataQuery->retrieveMetadata();
+
+		// order by metadata photo_taken
+		$metadataQuery->leftJoinIndex('photo_taken');
+		$query->orderBy($metadataQuery->getMetadataValueIntField(), 'desc');
+
+		return $metadataQuery;
+	}
+
 	/**
 	 * Perform a file system search in multiple caches
 	 *
@@ -175,20 +177,24 @@ public function searchInCaches(ISearchQuery $searchQuery, array $caches): array
 		$query = $builder->selectFileCache('file', false);
 
 		$requestedFields = $this->searchBuilder->extractRequestedFields($searchQuery->getSearchOperation());
+
 		if (in_array('systemtag', $requestedFields)) {
 			$this->equipQueryForSystemTags($query, $this->requireUser($searchQuery));
 		}
 		if (in_array('tagname', $requestedFields) || in_array('favorite', $requestedFields)) {
 			$this->equipQueryForDavTags($query, $this->requireUser($searchQuery));
 		}
 
+		$metadataQuery = $this->equipQueryForMetadata($query, $searchQuery);
 		$this->applySearchConstraints($query, $searchQuery, $caches);
 
 		$result = $query->execute();
 		$files = $result->fetchAll();
 
-		$rawEntries = array_map(function (array $data) {
-			return Cache::cacheEntryFromData($data, $this->mimetypeLoader);
+		$rawEntries = array_map(function (array $data) use ($metadataQuery) {
+			$entry = Cache::cacheEntryFromData($data, $this->mimetypeLoader);
+			$entry['metadata'] = $metadataQuery?->extractMetadata($data);
+			return $entry;
 		}, $files);
 
 		$result->closeCursor();

lib/private/FilesMetadata/Event/MetadataEventBase.php

@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2023 Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\FilesMetadata\Event;
+
+use OCP\Files\Node;
+use OCP\EventDispatcher\Event;
+use OCP\FilesMetadata\Model\IFilesMetadata;
+
+class MetadataEventBase extends Event {
+
+	public function __construct(
+		protected Node $node,
+		protected IFilesMetadata $metadata
+	) {
+		parent::__construct();
+	}
+
+	/**
+	 * returns id of the file
+	 *
+	 * @return Node
+	 * @since 28.0.0
+	 */
+	public function getNode(): Node {
+		return $this->node;
+	}
+
+	/**
+	 * returns Metadata model linked to file id, with already known metadata from the database.
+	 * If the object is modified using its setters, metadata are updated in database at the end of the event.
+	 *
+	 * @return IFilesMetadata
+	 * @since 28.0.0
+	 */
+	public function getMetadata(): IFilesMetadata {
+		return $this->metadata;
+	}
+}