From c826af427f571bce28a3f9ff9b47dc10d309c65b Mon Sep 17 00:00:00 2001 From: Louis Chemineau Date: Thu, 22 Feb 2024 17:41:16 +0100 Subject: [PATCH] Move oc_file_metadata.metadata migration to a background job Signed-off-by: Louis Chemineau Signed-off-by: Max --- core/BackgroundJobs/MetadataMigrationJob.php | 102 ++++++++++++++++++ .../Version27000Date20230309104325.php | 26 ++--- .../Version27000Date20230309104802.php | 14 +-- lib/composer/composer/LICENSE | 2 + lib/composer/composer/autoload_classmap.php | 2 + lib/composer/composer/autoload_static.php | 2 + lib/private/Repair.php | 16 +-- .../Repair/AddMetadataMigrationJob.php | 70 ++++++++++++ 8 files changed, 207 insertions(+), 27 deletions(-) create mode 100644 core/BackgroundJobs/MetadataMigrationJob.php create mode 100644 lib/private/Repair/AddMetadataMigrationJob.php diff --git a/core/BackgroundJobs/MetadataMigrationJob.php b/core/BackgroundJobs/MetadataMigrationJob.php new file mode 100644 index 0000000000000..0e88c31318ed1 --- /dev/null +++ b/core/BackgroundJobs/MetadataMigrationJob.php @@ -0,0 +1,102 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Core\BackgroundJobs; + +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; +use OCP\BackgroundJob\TimedJob; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +// Migrate oc_file_metadata.metadata to oc_file_metadata.value. +// This was previously done in a migration, but it is taking to much time in large instances. +// This job will progressively migrate the data 1 hour per night every night. +// Once done, it will remove itself from the job list. +class MetadataMigrationJob extends TimedJob { + public function __construct( + ITimeFactory $time, + private IDBConnection $db, + private IJobList $jobList, + ) { + parent::__construct($time); + + $this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE); + $this->setInterval(24 * 3600); + } + + protected function run(mixed $argument): void { + if (!$this->db->createSchema()->getTable('oc_file_metadata')->hasColumn('metadata')) { + return; + } + + $updateQuery = $this->db->getQueryBuilder(); + $updateQuery->update('file_metadata') + ->set('value', $updateQuery->createParameter('value')) + ->set('metadata', $updateQuery->createParameter('metadata')) + ->where($updateQuery->expr()->eq('id', $updateQuery->createParameter('id'))) + ->andWhere($updateQuery->expr()->eq('group_name', $updateQuery->createParameter('group_name'))); + + $selectQuery = $this->db->getQueryBuilder(); + $selectQuery->select('id', 'group_name', 'metadata') + ->from('file_metadata') + ->where($selectQuery->expr()->nonEmptyString('metadata')) + ->setMaxResults(1000); + + $movedRows = 0; + $startTime = time(); + + do { + // Stop if execution time is more than one hour. + if (time() - $startTime > 3600) { + return; + } + $movedRows = $this->chunkedCopying($updateQuery, $selectQuery); + } while ($movedRows !== 0); + + + $this->jobList->remove(MetadataMigrationJob::class); + } + + protected function chunkedCopying(IQueryBuilder $updateQuery, IQueryBuilder $selectQuery): int { + $this->db->beginTransaction(); + + $results = $selectQuery->executeQuery(); + + while ($row = $results->fetch()) { + $updateQuery + ->setParameter('id', (int)$row['id']) + ->setParameter('group_name', $row['group_name']) + ->setParameter('value', $row['metadata']) + ->setParameter('metadata', '') + ->executeStatement(); + } + + $results->closeCursor(); + $this->db->commit(); + + return $results->rowCount(); + } +} diff --git a/core/Migrations/Version27000Date20230309104325.php b/core/Migrations/Version27000Date20230309104325.php index e11b37b4b2988..0ecaa4a73a71d 100644 --- a/core/Migrations/Version27000Date20230309104325.php +++ b/core/Migrations/Version27000Date20230309104325.php @@ -72,19 +72,19 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt * @param array $options * @return void */ - public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) { - /** @var ISchemaWrapper $schema */ - $schema = $schemaClosure(); - $metadataTable = $schema->getTable('file_metadata'); + // public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) { + // /** @var ISchemaWrapper $schema */ + // $schema = $schemaClosure(); + // $metadataTable = $schema->getTable('file_metadata'); - if (!$metadataTable->hasColumn('metadata')) { - return; - } + // if (!$metadataTable->hasColumn('metadata')) { + // return; + // } - $this->connection - ->getQueryBuilder() - ->update('file_metadata') - ->set('value', 'metadata') - ->executeStatement(); - } + // $this->connection + // ->getQueryBuilder() + // ->update('file_metadata') + // ->set('value', 'metadata') + // ->executeStatement(); + // } } diff --git a/core/Migrations/Version27000Date20230309104802.php b/core/Migrations/Version27000Date20230309104802.php index 4bd50fe03962c..260ae83d9708b 100644 --- a/core/Migrations/Version27000Date20230309104802.php +++ b/core/Migrations/Version27000Date20230309104802.php @@ -43,14 +43,14 @@ class Version27000Date20230309104802 extends SimpleMigrationStep { * @return null|ISchemaWrapper */ public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper { - /** @var ISchemaWrapper $schema */ - $schema = $schemaClosure(); - $metadataTable = $schema->getTable('file_metadata'); + // /** @var ISchemaWrapper $schema */ + // $schema = $schemaClosure(); + // $metadataTable = $schema->getTable('file_metadata'); - if ($metadataTable->hasColumn('metadata')) { - $metadataTable->dropColumn('metadata'); - return $schema; - } + // if ($metadataTable->hasColumn('metadata')) { + // $metadataTable->dropColumn('metadata'); + // return $schema; + // } return null; } diff --git a/lib/composer/composer/LICENSE b/lib/composer/composer/LICENSE index 62ecfd8d0046b..f27399a042d95 100644 --- a/lib/composer/composer/LICENSE +++ b/lib/composer/composer/LICENSE @@ -1,3 +1,4 @@ + Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy @@ -17,3 +18,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index e4e2aaf3aa367..f455e9b010eef 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -895,6 +895,7 @@ 'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => $baseDir . '/core/BackgroundJobs/CheckForUserCertificates.php', 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MetadataMigrationJob' => $baseDir . '/core/BackgroundJobs/MetadataMigrationJob.php', 'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php', @@ -1476,6 +1477,7 @@ 'OC\\RepairException' => $baseDir . '/lib/private/RepairException.php', 'OC\\Repair\\AddBruteForceCleanupJob' => $baseDir . '/lib/private/Repair/AddBruteForceCleanupJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', + 'OC\\Repair\\AddMetadataMigrationJob' => $baseDir . '/lib/private/Repair/AddMetadataMigrationJob.php', 'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php', 'OC\\Repair\\ClearFrontendCaches' => $baseDir . '/lib/private/Repair/ClearFrontendCaches.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 1b448946a42c4..37affb1babc22 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -928,6 +928,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CheckForUserCertificates.php', 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MetadataMigrationJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MetadataMigrationJob.php', 'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php', @@ -1509,6 +1510,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\RepairException' => __DIR__ . '/../../..' . '/lib/private/RepairException.php', 'OC\\Repair\\AddBruteForceCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddBruteForceCleanupJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', + 'OC\\Repair\\AddMetadataMigrationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataMigrationJob.php', 'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php', 'OC\\Repair\\ClearFrontendCaches' => __DIR__ . '/../../..' . '/lib/private/Repair/ClearFrontendCaches.php', diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 330fa241b1e09..114afb3ead5f2 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -34,18 +34,13 @@ */ namespace OC; -use OC\Repair\CleanUpAbandonedApps; -use OCP\AppFramework\QueryException; -use OCP\AppFramework\Utility\ITimeFactory; -use OCP\Collaboration\Resources\IManager; -use OCP\EventDispatcher\IEventDispatcher; -use OCP\Migration\IOutput; -use OCP\Migration\IRepairStep; use OC\DB\Connection; use OC\DB\ConnectionAdapter; use OC\Repair\AddBruteForceCleanupJob; use OC\Repair\AddCleanupUpdaterBackupsJob; +use OC\Repair\AddMetadataMigrationJob; use OC\Repair\CleanTags; +use OC\Repair\CleanUpAbandonedApps; use OC\Repair\ClearFrontendCaches; use OC\Repair\ClearGeneratedAvatarCache; use OC\Repair\Collation; @@ -85,6 +80,12 @@ use OC\Repair\RepairMimeTypes; use OC\Repair\SqliteAutoincrement; use OC\Template\JSCombiner; +use OCP\AppFramework\QueryException; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\Collaboration\Resources\IManager; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; use Psr\Log\LoggerInterface; use Throwable; @@ -210,6 +211,7 @@ public static function getRepairSteps(): array { \OCP\Server::get(AddTokenCleanupJob::class), \OCP\Server::get(CleanUpAbandonedApps::class), \OCP\Server::get(AddMissingSecretJob::class), + \OCP\Server::get(AddRemoveOldTasksBackgroundJob::class), ]; } diff --git a/lib/private/Repair/AddMetadataMigrationJob.php b/lib/private/Repair/AddMetadataMigrationJob.php new file mode 100644 index 0000000000000..7bc3573a1a276 --- /dev/null +++ b/lib/private/Repair/AddMetadataMigrationJob.php @@ -0,0 +1,70 @@ + + * + * @author Louis Chmn + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OC\Repair; + +use OC\Core\BackgroundJobs\MetadataMigrationJob; +use OCP\BackgroundJob\IJobList; +use OCP\IDBConnection; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class AddMetadataMigrationJob implements IRepairStep { + public function __construct( + private IJobList $jobList, + private IDBConnection $db, + ) { + } + + public function getName() { + return 'Queue a job to migrate the file_metadata table and delete the metadata column once empty'; + } + + public function run(IOutput $output) { + $schema = $this->db->createSchema(); + $metadataTable = $schema->getTable('oc_file_metadata'); + + if (!$metadataTable->hasColumn('metadata')) { + return; + } + + $selectQuery = $this->db->getQueryBuilder(); + $result = $selectQuery->select('id', 'group_name', 'metadata') + ->from('file_metadata') + ->where($selectQuery->expr()->nonEmptyString('metadata')) + ->setMaxResults(1) + ->executeQuery(); + + if ($result->rowCount() === 0) { + $output->info('Removing metadata column from the file_metadata table.'); + $metadataTable->dropColumn('metadata'); + $this->db->migrateToSchema($schema); + return; + } + + if ($this->jobList->has(MetadataMigrationJob::class, null)) { + return; + } + + $this->jobList->add(MetadataMigrationJob::class); + } +}