From 170f8d50d0d5896623c3a6a09016a88a4a457aaf Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 10 Sep 2024 10:27:13 +0200 Subject: [PATCH] Initial Push for Batch Deletes --- src/Database/Adapter.php | 10 +++ src/Database/Adapter/MariaDB.php | 55 ++++++++++++ src/Database/Adapter/Mongo.php | 13 +++ src/Database/Adapter/Postgres.php | 55 ++++++++++++ src/Database/Database.php | 78 ++++++++++++++++ tests/e2e/Adapter/Base.php | 145 ++++++++++++++++++++++++++++++ 6 files changed, 356 insertions(+) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 4c2aa326a..bf010faaa 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -587,6 +587,16 @@ abstract public function updateDocuments(string $collection, array $documents, i */ abstract public function deleteDocument(string $collection, string $id): bool; + /** + * Delete Documents + * + * @param string $collection + * @param array<\Utopia\Database\Query> $queries + * + * @return bool + */ + abstract public function deleteDocuments(string $collection, array $queries): bool; + /** * Find Documents * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index aaaa260fe..6256fec37 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1635,6 +1635,61 @@ public function deleteDocument(string $collection, string $id): bool return $deleted; } + /** + * Delete Documents + * + * @param string $collection + * @param array<\Utopia\Database\Query> $queries + * + * @return bool + */ + public function deleteDocuments(string $collection, array $queries): bool + { + $name = $this->filter($collection); + $where = []; + + $queries = array_map(fn ($query) => clone $query, $queries); + + $conditions = $this->getSQLConditions($queries); + if(!empty($conditions)) { + $where[] = $conditions; + } + + if ($this->sharedTables) { + $where[] = "table_main._tenant = :_tenant"; + } + + $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : ''; + + $selections = $this->getAttributeSelections($queries); + + $sql = " + USE {$this->database}; + DELETE {$this->getAttributeProjection($selections, 'table_main')} + FROM {$this->getSQLTable($name)} as table_main + {$sqlWhere}; + "; + + $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql); + + $stmt = $this->getPDO()->prepare($sql); + + foreach ($queries as $query) { + $this->bindConditionValue($stmt, $query); + } + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + + try { + $stmt->execute(); + } catch (PDOException $e) { + $this->processException($e); + } + + return true; + } + /** * Find Documents * diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index feb1d9a15..62b411ca3 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -909,6 +909,19 @@ public function deleteDocument(string $collection, string $id): bool return (!!$result); } + /** + * Delete Documents + * + * @param string $collection + * @param array<\Utopia\Database\Query> $queries + * + * @return bool + */ + public function deleteDocuments(string $collection, array $queries): bool + { + throw new \Exception('Not Implemented'); + } + /** * Update Attribute. * @param string $collection diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 8c25731d6..7ea145f85 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1558,6 +1558,61 @@ public function deleteDocument(string $collection, string $id): bool return $deleted; } + + /** + * Delete Documents + * + * @param string $collection + * @param array<\Utopia\Database\Query> $queries + * + * @return bool + */ + public function deleteDocuments(string $collection, array $queries): bool + { + $name = $this->filter($collection); + $where = []; + + $queries = array_map(fn ($query) => clone $query, $queries); + + $conditions = $this->getSQLConditions($queries); + if(!empty($conditions)) { + $where[] = $conditions; + } + + if ($this->sharedTables) { + $where[] = "table_main._tenant = :_tenant"; + } + + $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : ''; + + $selections = $this->getAttributeSelections($queries); + + $sql = " + SELECT {$this->getAttributeProjection($selections, 'table_main')} + FROM {$this->getSQLTable($name)} as table_main + {$sqlWhere} + "; + + $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql); + + $stmt = $this->getPDO()->prepare($sql); + + foreach ($queries as $query) { + $this->bindConditionValue($stmt, $query); + } + if ($this->sharedTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + + try { + $stmt->execute(); + } catch (PDOException $e) { + $this->processException($e); + } + + return true; + } + /** * Find Documents * diff --git a/src/Database/Database.php b/src/Database/Database.php index 437f18881..58b9d0000 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4944,6 +4944,84 @@ private function deleteCascade(Document $collection, Document $relatedCollection } } + /** + * Delete Documents + * + * Deletes all documents which match the given query, will respect the relationship's onDelete optin. + * + * @param string $collection + * @param array $queries + * + * @return bool + * + * @throws AuthorizationException + * @throws DatabaseException + * @throws RestrictedException + */ + public function deleteDocuments(string $collection, array $queries = []): bool + { + if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + + $collection = $this->silent(fn () => $this->getCollection($collection)); + + $deleted = $this->withTransaction(function () use ($collection, $queries) { + $lastDocument = null; + while (true) { + $affectedDocuments = $this->find($collection->getId(), array_merge( + empty($lastDocument) ? [ + Query::limit(100), + ] : [ + Query::limit(100), + Query::cursorAfter($lastDocument), + ], + $queries, + )); + + if (empty($affectedDocuments)) { + break; + } + + foreach ($affectedDocuments as $document) { + $validator = new Authorization(self::PERMISSION_DELETE); + + if ($collection->getId() !== self::METADATA) { + $documentSecurity = $collection->getAttribute('documentSecurity', false); + if (!$validator->isValid([ + ...$collection->getDelete(), + ...($documentSecurity ? $document->getDelete() : []) + ])) { + throw new AuthorizationException($validator->getDescription()); + } + } + + // Delete Relationships + if ($this->resolveRelationships) { + $document = $this->silent(fn () => $this->deleteDocumentRelationships($collection, $document)); + } + + // Fire events + $this->trigger(self::EVENT_DOCUMENT_DELETE, $document); + + $this->purgeRelatedDocuments($collection, $document->getId()); + $this->purgeCachedDocument($collection->getId(), $document->getId()); + } + + if (count($affectedDocuments) < 100) { + break; + } else { + $lastDocument = end($affectedDocuments); + } + } + + // Mass delete using adapter with query + return $this->adapter->deleteDocuments($collection->getId(), $queries); + }); + + return $deleted; + } + /** * Cleans the all the collection's documents from the cache * And the all related cached documents. diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 72587d44a..ff2998633 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -15608,4 +15608,149 @@ public function testEvents(): void $database->delete('hellodb'); }); } + + public function propegateBulkDocuments(bool $documentSecurity = false): void + { + for ($i = 0; $i < 10; $i++) { + var_dump(new Document( + array_merge([ + '$id' => 'doc' . $i, + 'text' => 'value' . $i, + 'integer' => $i + ], $documentSecurity ? [ + '$permissions' => [ + Permission::delete(Role::any()), + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ] : []) + )); + + static::getDatabase()->createDocument('bulk_delete', new Document( + array_merge([ + '$id' => 'doc' . $i, + 'text' => 'value' . $i, + 'integer' => $i + ], $documentSecurity ? [ + '$permissions' => [ + Permission::create(Role::any()), + Permission::read(Role::any()), + ], + ] : []) + )); + } + } + + public function testDeleteBulkDocuments(): void + { + static::getDatabase()->createCollection( + 'bulk_delete', + attributes: [ + new Document([ + '$id' => 'text', + 'type' => Database::VAR_STRING, + 'size' => 100, + 'required' => true, + ]), + new Document([ + '$id' => 'integer', + 'type' => Database::VAR_INTEGER, + 'size' => 10, + 'required' => true, + ]) + ], + documentSecurity: false, + permissions: [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::delete(Role::any()) + ] + ); + + $this->propegateBulkDocuments(); + + $docs = static::getDatabase()->find('bulk_delete'); + $this->assertCount(10, $docs); + + // TEST: Bulk Delete All Documents + static::getDatabase()->deleteDocuments('bulk_delete'); + + $docs = static::getDatabase()->find('bulk_delete'); + $this->assertCount(0, $docs); + + // TEST: Bulk delete documents with queries. + $this->propegateBulkDocuments(); + + static::getDatabase()->deleteDocuments('bulk_delete', [ + Query::greaterThanEqual('integer', 5) + ]); + + $docs = static::getDatabase()->find('bulk_delete'); + $this->assertCount(5, $docs); + + // TEST (FAIL): Bulk delete all documents with invalid collection permission + static::getDatabase()->updateCollection('bulk_delete', [], false); + try { + static::getDatabase()->deleteDocuments('bulk_delete'); + $this->fail('Bulk deleted documents with invalid collection permission'); + } catch (\Utopia\Database\Exception\Authorization) { + } + + static::getDatabase()->updateCollection('bulk_delete', [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::delete(Role::any()) + ], false); + static::getDatabase()->deleteDocuments('bulk_delete'); + $this->assertEquals(0, count($this->getDatabase()->find('bulk_delete'))); + + // TEST (FAIL): Bulk delete all documents with invalid document permissions + // Authorization::setRole(Role::any()->toString()); + static::getDatabase()->updateCollection('bulk_delete', [ + Permission::create(Role::any()), + ], true); + $this->propegateBulkDocuments(true); + + try { + static::getDatabase()->deleteDocuments('bulk_delete'); + $this->fail('Bulk deleted documents with invalid document permission'); + } catch (\Utopia\Database\Exception\Authorization) { + } + + static::getDatabase()->updateCollection('bulk_delete', [ + Permission::create(Role::any()), + Permission::read(Role::any()), + Permission::delete(Role::any()) + ], false); + static::getDatabase()->deleteDocuments('bulk_delete'); + $this->assertEquals(0, count($this->getDatabase()->find('bulk_delete'))); + + // Teardown + static::getDatabase()->deleteCollection('bulk_delete'); + } + + + public function testDeleteBulkDocumentsRelationships(): void + { + if (!static::getDatabase()->getAdapter()->getSupportForRelationships()) { + $this->expectNotToPerformAssertions(); + return; + } + + static::getDatabase()->createCollection('bulk_delete_r_1'); + static::getDatabase()->createCollection('bulk_delete_r_2'); + + // ONE_TO_ONE + static::getDatabase()->createRelationship( + collection: 'bulk_delete_r_1', + relatedCollection: 'bulk_delete_r_2', + type: Database::RELATION_ONE_TO_ONE, + ); + + // Restrict + + // NULL + + // Cascade + } }