Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
eldertek committed Dec 16, 2024
1 parent 76582f3 commit 84f4c0c
Show file tree
Hide file tree
Showing 7 changed files with 61 additions and 193 deletions.
15 changes: 1 addition & 14 deletions lib/BackgroundJob/CleanUpDB.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,28 +62,15 @@ protected function run($argument): void
// Clean up any unhandled delete or rename events
$fileInfos = $this->fileInfoService->findAll();

$batchSize = 100; // Define the batch size
$batch = [];

foreach ($fileInfos as $fileInfo) {
try {
$this->folderService->getNodeByFileInfo($fileInfo);
} catch (NotFoundException $e) {
$this->logger->info('FileInfo ' . $fileInfo->getPath() . ' will be deleted');
$batch[] = $fileInfo;

if (count($batch) >= $batchSize) {
$this->fileInfoService->deleteBatch($batch);
$batch = [];
}
$this->fileInfoService->delete($fileInfo);
}
}

// Delete any remaining fileInfos in the batch
if (!empty($batch)) {
$this->fileInfoService->deleteBatch($batch);
}

unset($fileInfo);
}
}
28 changes: 4 additions & 24 deletions lib/BackgroundJob/FindDuplicates.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
use OCP\IUser;
use OCP\BackgroundJob\TimedJob;
use Psr\Log\LoggerInterface;
use Doctrine\DBAL\Exception\DriverException;

class FindDuplicates extends TimedJob
{
Expand Down Expand Up @@ -75,28 +74,9 @@ public function __construct(
*/
protected function run($argument): void
{
// Fetch all users in a single query
$users = $this->userManager->search('');

// Process users in batches
$batchSize = 100;
$batches = array_chunk($users, $batchSize);

foreach ($batches as $batch) {
foreach ($batch as $user) {
try {
$this->fileInfoService->scanFiles($user->getUID());
} catch (DriverException $e) {
if ($e->getSQLState() === '25P02') {
$this->logger->error('SQLSTATE[25P02]: In failed sql transaction: ' . $e->getMessage(), ['app' => 'duplicatefinder', 'exception' => $e]);
// Handle the error, possibly by rolling back the transaction or taking other actions
} else {
$this->logger->error('An error occurred during scanning.', ['app' => 'duplicatefinder', 'exception' => $e]);
}
} catch (\Exception $e) {
$this->logger->error('An error occurred during scanning.', ['app' => 'duplicatefinder', 'exception' => $e]);
}
}
}
// Call the scanFiles method for all users using the userManager.
$this->userManager->callForAllUsers(function (IUser $user): void {
$this->fileInfoService->scanFiles($user->getUID());
});
}
}
18 changes: 2 additions & 16 deletions lib/Command/FindDuplicates.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Doctrine\DBAL\Exception\DriverException;

class FindDuplicates extends Command
{
Expand Down Expand Up @@ -185,19 +184,6 @@ private function findDuplicates(string $user, array $paths): void
}
}

try {
CMDUtils::showDuplicates($this->fileDuplicateService, $this->output, $callback, $user);
} catch (DriverException $e) {
if ($e->getSQLState() === '25P02') {
$this->logger->error('SQLSTATE[25P02]: In failed sql transaction: ' . $e->getMessage(), ['app' => 'duplicatefinder', 'exception' => $e]);
$this->output->writeln('<error>SQLSTATE[25P02]: In failed sql transaction: ' . $e->getMessage() . '</error>');
} else {
$this->logger->error('An error occurred during scanning.', ['app' => 'duplicatefinder', 'exception' => $e]);
$this->output->writeln('<error>An error occurred during scanning.</error>');
}
} catch (\Exception $e) {
$this->logger->error('An error occurred during scanning.', ['app' => 'duplicatefinder', 'exception' => $e]);
$this->output->writeln('<error>An error occurred during scanning.</error>');
}
CMDUtils::showDuplicates($this->fileDuplicateService, $this->output, $callback, $user);
}
}
}
124 changes: 37 additions & 87 deletions lib/Db/FileDuplicateMapper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,32 @@

use OCP\IDBConnection;
use OCP\DB\QueryBuilder\IQueryBuilder;
use Psr\Log\LoggerInterface;
use Doctrine\DBAL\Platforms\PostgreSQL\PostgreSQLPlatform;
use OCP\ILogger;

/**
* @extends EQBMapper<FileDuplicate>
*/
class FileDuplicateMapper extends EQBMapper
{
/** @var LoggerInterface */
/** @var ILogger */
private $logger;

public function __construct(IDBConnection $db, LoggerInterface $logger)
public function __construct(IDBConnection $db, ILogger $logger)
{
parent::__construct($db, 'duplicatefinder_dups', FileDuplicate::class);
$this->logger = $logger;
}

/**
* Find a duplicate by hash and type
*
* @param string $hash The hash to search for
* @param string $type The type of hash (defaults to 'file_hash')
* @return FileDuplicate
* @throws \OCP\AppFramework\Db\DoesNotExistException
* @throws \OCP\AppFramework\Db\MultipleObjectsReturnedException
*/
public function find(string $hash, string $type = 'file_hash'): FileDuplicate
{
try {
$qb = $this->db->getQueryBuilder();
$qb->select('*')
->from($this->getTableName())
->where(
$qb->expr()->eq('hash', $qb->createNamedParameter($hash)),
$qb->expr()->eq('type', $qb->createNamedParameter($type))
);
return $this->findEntity($qb);
} catch (\Exception $e) {
$this->logger->error('Error in find: ' . $e->getMessage(), ['app' => 'duplicatefinder']);
throw $e;
}
$qb = $this->db->getQueryBuilder();
$qb->select('*')
->from($this->getTableName())
->where(
$qb->expr()->eq('hash', $qb->createNamedParameter($hash)),
$qb->expr()->eq('type', $qb->createNamedParameter($type))
);
return $this->findEntity($qb);
}

/**
Expand All @@ -53,67 +38,38 @@ public function find(string $hash, string $type = 'file_hash'): FileDuplicate
* @param int|null $offset
* @param array<array<string>> $orderBy
* @return array<FileDuplicate>
* @throws \OCP\AppFramework\Db\MultipleObjectsReturnedException
* @throws \OCP\Files\NotFoundException
* @throws \OCP\Files\NotPermittedException
*/
public function findAll(
?string $user = null,
?int $limit = null,
?int $offset = null,
?array $orderBy = [['hash'], ['type']]
): array {
try {
$qb = $this->db->getQueryBuilder();
$qb->select('d.id as id', 'd.type', 'd.hash', 'd.acknowledged')
->from($this->getTableName(), 'd');

if ($user !== null) {
$qb->andWhere($qb->expr()->eq('user_id', $qb->createNamedParameter($user)));
}
$qb = $this->db->getQueryBuilder();
$qb->select('d.id as id', 'd.type', 'd.hash', 'd.acknowledged')
->from($this->getTableName(), 'd');

if ($limit !== null) {
$qb->setMaxResults($limit);
}
if ($offset !== null) {
$qb->setFirstResult($offset);
}
if ($limit !== null) {
$qb->setMaxResults($limit); // Set the limit of rows to fetch
}
if ($offset !== null) {
$qb->setFirstResult($offset); // Set the offset to start fetching rows
}

if ($orderBy !== null) {
foreach ($orderBy as $order) {
$qb->addOrderBy($order[0], isset($order[1]) ? $order[1] : null);
}
unset($order);
if ($orderBy !== null) {
foreach ($orderBy as $order) {
$qb->addOrderBy($order[0], isset($order[1]) ? $order[1] : null);
}
// Handle PostgreSQL-specific SQL syntax issues
if ($this->db->getDatabasePlatform() instanceof \Doctrine\DBAL\Platforms\PostgreSQLPlatform) {
$qb->addSelect('pg_column_size("d"."hash") as hash_size');
}

return $this->findEntities($qb);
} catch (\Exception $e) {
$this->logger->error('Error in findAll: ' . $e->getMessage(), ['app' => 'duplicatefinder']);
throw $e;
unset($order);
}
return $this->findEntities($qb);
}

/**
* Clears the duplicate entries from the database
*
* @param string|null $table Optional specific table to clear
* @throws \OCP\AppFramework\Db\DoesNotExistException
*/
public function clear(?string $table = null): void
{
try {
parent::clear($this->getTableName() . '_f');
parent::clear();
} catch (\Exception $e) {
$this->logger->error('Error in clear: ' . $e->getMessage(), ['app' => 'duplicatefinder']);
throw $e;
}
parent::clear($this->getTableName() . '_f');
parent::clear();
}

/**
* Marks the specified duplicate as acknowledged.
*
Expand All @@ -128,15 +84,16 @@ public function markAsAcknowledged(string $hash): bool
$qb->update($this->getTableName())
->set('acknowledged', $qb->createNamedParameter(true, IQueryBuilder::PARAM_BOOL))
->where($qb->expr()->eq('hash', $qb->createNamedParameter($hash)))
->executeStatement();
->execute();

return true;
} catch (\Exception $e) {
$this->logger->error('Error in markAsAcknowledged: ' . $e->getMessage(), ['app' => 'duplicatefinder']);
$this->logger->error($e->getMessage());
return false;
}
}


/**
* Removes the acknowledged status from the specified duplicate.
*
Expand All @@ -151,7 +108,7 @@ public function unmarkAcknowledged(string $hash): bool
$qb->update($this->getTableName())
->set('acknowledged', $qb->createNamedParameter(false, IQueryBuilder::PARAM_BOOL))
->where($qb->expr()->eq('hash', $qb->createNamedParameter($hash)))
->executeStatement();
->execute();

return true;
} catch (\Exception $e) {
Expand All @@ -170,30 +127,23 @@ public function getTotalCount(string $type = 'unacknowledged'): int
{
$qb = $this->db->getQueryBuilder();

// Start with a basic SELECT COUNT query
$qb->select($qb->func()->count('*', 'total_count'))
->from($this->getTableName());

// Add conditions based on the type
if ($type === 'acknowledged') {
$qb->where($qb->expr()->eq('acknowledged', $qb->createNamedParameter(true, IQueryBuilder::PARAM_BOOL)));
} elseif ($type === 'unacknowledged') {
$qb->where($qb->expr()->eq('acknowledged', $qb->createNamedParameter(false, IQueryBuilder::PARAM_BOOL)));
}
} // No condition needed for 'all', as we want to count all rows

$result = $qb->executeQuery();
// Execute the query and fetch the result
$result = $qb->execute();
$row = $result->fetch();
$result->closeCursor();

// Return the count result as an integer
return (int) ($row ? $row['total_count'] : 0);
}

/**
* Handle byte sequences for UTF8 encoding.
*
* @param string $input The input string.
* @return string The sanitized string.
*/
public function handleUTF8ByteSequences(string $input): string
{
return mb_convert_encoding($input, 'UTF-8', 'UTF-8');
}
}
40 changes: 5 additions & 35 deletions lib/Service/ConfigService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,6 @@ class ConfigService
/** @var LoggerInterface */
private $logger;

/** @var int|null */
private $findJobInterval = null;
/** @var int|null */
private $cleanupJobInterval = null;
/** @var bool|null */
private $filesystemEventsDisabled = null;
/** @var bool|null */
private $mountedFilesIgnored = null;
/** @var string|null */
private $installedVersion = null;

public function __construct(
IConfig $config,
LoggerInterface $logger
Expand Down Expand Up @@ -73,65 +62,46 @@ public function setUserValue(string $userId, string $key, string $value) : void

public function getFindJobInterval() : int
{
if ($this->findJobInterval === null) {
$this->findJobInterval = $this->getIntVal('backgroundjob_interval_find', 60*60*24*5);
}
return $this->findJobInterval;
return $this->getIntVal('backgroundjob_interval_find', 60*60*24*5);
}

public function getCleanupJobInterval() : int
{
if ($this->cleanupJobInterval === null) {
$this->cleanupJobInterval = $this->getIntVal('backgroundjob_interval_cleanup', 60*60*24*2);
}
return $this->cleanupJobInterval;
return $this->getIntVal('backgroundjob_interval_cleanup', 60*60*24*2);
}

public function areFilesytemEventsDisabled():bool
{
if ($this->filesystemEventsDisabled === null) {
$this->filesystemEventsDisabled = $this->getBoolVal('disable_filesystem_events', false);
}
return $this->filesystemEventsDisabled;
return $this->getBoolVal('disable_filesystem_events', false);
}

public function areMountedFilesIgnored() : bool
{
if ($this->mountedFilesIgnored === null) {
$this->mountedFilesIgnored = $this->getBoolVal('ignore_mounted_files', false);
}
return $this->mountedFilesIgnored;
return $this->getBoolVal('ignore_mounted_files', false);
}

public function getInstalledVersion() : string
{
if ($this->installedVersion === null) {
$this->installedVersion = $this->config->getAppValue(Application::ID, 'installed_version', '0.0.0');
}
return $this->installedVersion;
return $this->config->getAppValue(Application::ID, 'installed_version', '0.0.0');
}

public function setFindJobInterval(int $value) : void
{
$this->setIntVal('backgroundjob_interval_find', $value);
$this->findJobInterval = $value;
}

public function setCleanupJobInterval(int $value) : void
{
$this->setIntVal('backgroundjob_interval_cleanup', $value);
$this->cleanupJobInterval = $value;
}

public function setFilesytemEventsDisabled(bool $value):void
{
$this->setBoolVal('disable_filesystem_events', $value);
$this->filesystemEventsDisabled = $value;
}

public function setMountedFilesIgnored(bool $value) : void
{
$this->setBoolVal('ignore_mounted_files', $value);
$this->mountedFilesIgnored = $value;
}
}
Loading

0 comments on commit 84f4c0c

Please sign in to comment.