From a7d7aee72951ff4e0b6b6b136daf4871436f1a48 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 6 Jun 2024 00:06:14 +1200
Subject: [PATCH 001/100] Add mirroring filter

---
 src/Database/Database.php         |  1 +
 src/Database/Mirroring/Filter.php | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 src/Database/Mirroring/Filter.php

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 0236b9d92..dfafb8874 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -915,6 +915,7 @@ public function list(): array
      *
      * @param string|null $database
      * @return bool
+     * @throws DatabaseException
      */
     public function delete(?string $database = null): bool
     {
diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
new file mode 100644
index 000000000..848dd7561
--- /dev/null
+++ b/src/Database/Mirroring/Filter.php
@@ -0,0 +1,29 @@
+<?php
+
+namespace Utopia\Database\Mirroring;
+
+use Utopia\Database\Database;
+use Utopia\Database\Document;
+
+abstract class Filter
+{
+    abstract public function onCreateDocument(
+        Database $source,
+        Database $destination,
+        string $collection,
+        Document $document,
+    ): Document;
+
+    abstract public function onUpdateDocument(
+        Database $source,
+        Database $destination,
+        string $collection,
+        Document $document,
+    ): Document;
+
+    abstract public function onDeleteDocument(
+        Database $source,
+        Database $destination,
+        string $collection,
+    ): void;
+}

From 61f066f891d1acd13188a9f9f4df4d0fa4eb7a25 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 6 Jun 2024 00:07:00 +1200
Subject: [PATCH 002/100] Add mirror database

---
 src/Database/Mirror.php | 422 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 422 insertions(+)
 create mode 100644 src/Database/Mirror.php

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
new file mode 100644
index 000000000..db2fe58c1
--- /dev/null
+++ b/src/Database/Mirror.php
@@ -0,0 +1,422 @@
+<?php
+
+namespace Utopia\Database;
+
+use Utopia\Cache\Cache;
+use Utopia\Database\Exception\Duplicate as DuplicateException;
+use Utopia\Database\Exception\Limit;
+use Utopia\Database\Helpers\ID;
+use Utopia\Database\Mirroring\Filter;
+use Utopia\Database\Validator\Authorization;
+
+class Mirror extends Database
+{
+    protected Database $destination;
+
+    /**
+     * Filters to apply to documents before writing to the destination database
+     *
+     * @var array<Filter>
+     */
+    protected array $writeFilters = [];
+
+    /**
+     * Collections that should only be present in the source database
+     */
+    protected const SOURCE_ONLY_COLLECTIONS = [
+        'upgrades',
+    ];
+
+    /**
+     * Callbacks to run when an error occurs on the destination database
+     *
+     * @var array<callable(string, \Throwable): void>
+     */
+    protected array $errorCallbacks = [];
+
+    /**
+     * @param Adapter $adapter
+     * @param Cache $cache
+     * @param Database $destination
+     * @param array<Filter> $filters
+     */
+    public function __construct(
+        Adapter $adapter,
+        Cache $cache,
+        Database $destination,
+        array $filters = [],
+    ) {
+        parent::__construct($adapter, $cache);
+        $this->destination = $destination;
+        $this->writeFilters = $filters;
+    }
+
+    public function getDestination(): ?Database
+    {
+        return $this->destination;
+    }
+
+    public function getSource(): Database
+    {
+        return $this;
+    }
+
+    /**
+     * @param callable(string, \Throwable): void $callback
+     * @return void
+     */
+    public function onError(callable $callback): void
+    {
+        $this->errorCallbacks[] = $callback;
+    }
+
+    /**
+     * @param string $method
+     * @param array<mixed> $args
+     * @return mixed
+     */
+    protected function delegate(string $method, array $args = []): mixed
+    {
+        $result = parent::{$method}(...$args);
+
+        try {
+            $result = $this->destination->{$method}(...$args);
+        } catch (\Throwable $err) {
+            $this->logError($method, $err);
+        }
+
+        return $result;
+    }
+
+    public function enableValidation(): self
+    {
+        return $this->delegate('enableValidation');
+    }
+
+    public function disableValidation(): self
+    {
+        return $this->delegate('disableValidation');
+    }
+
+    public function delete(?string $database = null): bool
+    {
+        return $this->delegate('delete', [$database]);
+    }
+
+    public function create(?string $database = null): bool
+    {
+        return $this->delegate('create', [$database]);
+    }
+
+    public function createCollection(string $id, array $attributes = [], array $indexes = [], array $permissions = null, bool $documentSecurity = true): Document
+    {
+        $result = parent::createCollection(
+            $id,
+            $attributes,
+            $indexes,
+            $permissions,
+            $documentSecurity
+        );
+
+        try {
+            $this->destination->createCollection(
+                $id,
+                $attributes,
+                $indexes,
+                $permissions,
+                $documentSecurity
+            );
+
+            $this->createUpgrades();
+
+            parent::createDocument('upgrades', new Document([
+                '$id' => $id,
+                'collectionId' => $id,
+                'status' => 'upgraded'
+            ]));
+        } catch (\Throwable $err) {
+            $this->logError('createCollection', $err);
+        }
+        return $result;
+    }
+
+    public function updateCollection(string $id, array $permissions, bool $documentSecurity): Document
+    {
+        return $this->delegate('updateCollection', [$id, $permissions, $documentSecurity]);
+    }
+
+    public function deleteCollection(string $id): bool
+    {
+        return $this->delegate('deleteCollection', [$id]);
+    }
+
+    public function createAttribute(string $collection, string $id, string $type, int $size, bool $required, $default = null, bool $signed = true, bool $array = false, string $format = null, array $formatOptions = [], array $filters = []): bool
+    {
+        return $this->delegate('createAttribute', [$collection, $id, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters]);
+    }
+
+    public function updateAttribute(string $collection, string $id, string $type = null, int $size = null, bool $required = null, mixed $default = null, bool $signed = null, bool $array = null, string $format = null, ?array $formatOptions = null, ?array $filters = null): Document
+    {
+        return $this->delegate('updateAttribute', [$collection, $id, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters]);
+    }
+
+    public function deleteAttribute(string $collection, string $id): bool
+    {
+        return $this->delegate('deleteAttribute', [$collection, $id]);
+    }
+
+    public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool
+    {
+        return $this->delegate('createIndex', [$collection, $id, $type, $attributes, $lengths, $orders]);
+    }
+
+    public function deleteIndex(string $collection, string $id): bool
+    {
+        return $this->delegate('deleteIndex', [$collection, $id]);
+    }
+
+    public function createDocument(string $collection, Document $document): Document
+    {
+        $document = parent::createDocument($collection, $document);
+
+        if (\in_array($collection, self::SOURCE_ONLY_COLLECTIONS)) {
+            return $document;
+        }
+
+        $upgrade = $this->getUpgradeStatus($collection);
+        if ($upgrade->getAttribute('status', '') !== 'upgraded') {
+            return $document;
+        }
+
+        try {
+            $clone = clone $document;
+
+            foreach ($this->writeFilters as $filter) {
+                $clone = $filter->onCreateDocument(
+                    source: $this,
+                    destination: $this->destination,
+                    collection: $collection,
+                    document: $clone,
+                );
+            }
+
+            $this->destination->setPreserveDates(true);
+            $this->destination->createDocument($collection, $clone);
+            $this->destination->setPreserveDates(false);
+        } catch (\Throwable $err) {
+            $this->logError('createDocument', $err);
+        }
+
+        return $document;
+    }
+
+    public function updateDocument(string $collection, string $id, Document $document): Document
+    {
+        $document = parent::updateDocument($collection, $id, $document);
+
+        $upgrade = $this->getUpgradeStatus($collection);
+        if ($upgrade->getAttribute('status', '') !== 'upgraded') {
+            return $document;
+        }
+
+        try {
+            $clone = clone $document;
+
+            foreach ($this->writeFilters as $filter) {
+                $clone = $filter->onUpdateDocument(
+                    source: $this,
+                    destination: $this->destination,
+                    collection: $collection,
+                    document: $clone,
+                );
+            }
+
+            if (!$this->destination->getDocument($collection, $id)->isEmpty()) {
+                $this->destination->setPreserveDates(true);
+                $this->destination->updateDocument($collection, $id, $clone);
+                $this->destination->setPreserveDates(false);
+            }
+        } catch (\Throwable $err) {
+            $this->logError('updateDocument', $err);
+        }
+
+        return $document;
+    }
+
+    public function deleteDocument(string $collection, string $id): bool
+    {
+        $result = parent::deleteDocument($collection, $id);
+
+        $upgrade = $this->getUpgradeStatus($collection);
+        if ($upgrade->getAttribute('status', '') !== 'upgraded') {
+            return $result;
+        }
+
+        try {
+            $this->destination->deleteDocument($collection, $id);
+
+            foreach ($this->writeFilters as $filter) {
+                $filter->onDeleteDocument(
+                    source: $this,
+                    destination: $this->destination,
+                    collection: $collection,
+                );
+            }
+        } catch (\Throwable $err) {
+            $this->logError('deleteDocument', $err);
+        }
+
+        return $result;
+    }
+
+    public function updateAttributeRequired(string $collection, string $id, bool $required): Document
+    {
+        return $this->delegate('updateAttributeRequired', [$collection, $id, $required]);
+    }
+
+    public function updateAttributeFormat(string $collection, string $id, string $format): Document
+    {
+        return $this->delegate('updateAttributeFormat', [$collection, $id, $format]);
+    }
+
+    public function updateAttributeFormatOptions(string $collection, string $id, array $formatOptions): Document
+    {
+        return $this->delegate('updateAttributeFormatOptions', [$collection, $id, $formatOptions]);
+    }
+
+    public function updateAttributeFilters(string $collection, string $id, array $filters): Document
+    {
+        return $this->delegate('updateAttributeFilters', [$collection, $id, $filters]);
+    }
+
+    public function updateAttributeDefault(string $collection, string $id, mixed $default = null): Document
+    {
+        return $this->delegate('updateAttributeDefault', [$collection, $id, $default]);
+    }
+
+    public function renameAttribute(string $collection, string $old, string $new): bool
+    {
+        return $this->delegate('renameAttribute', [$collection, $old, $new]);
+    }
+
+    public function createRelationship(
+        string $collection,
+        string $relatedCollection,
+        string $type,
+        bool $twoWay = false,
+        ?string $id = null,
+        ?string $twoWayKey = null,
+        string $onDelete = Database::RELATION_MUTATE_RESTRICT
+    ): bool {
+        return $this->delegate('createRelationship', [$collection, $relatedCollection, $type, $twoWay, $id, $twoWayKey, $onDelete]);
+    }
+
+    public function updateRelationship(
+        string $collection,
+        string $id,
+        ?string $newKey = null,
+        ?string $newTwoWayKey = null,
+        ?bool $twoWay = null,
+        ?string $onDelete = null
+    ): bool {
+        return $this->delegate('updateRelationship', [$collection, $id, $newKey, $newTwoWayKey, $twoWay, $onDelete]);
+    }
+
+    public function deleteRelationship(string $collection, string $id): bool
+    {
+        return $this->delegate('deleteRelationship', [$collection, $id]);
+    }
+
+
+    public function renameIndex(string $collection, string $old, string $new): bool
+    {
+        return $this->delegate('renameIndex', [$collection, $old, $new]);
+    }
+
+    public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $max = null): bool
+    {
+        return $this->delegate('increaseDocumentAttribute', [$collection, $id, $attribute, $value, $max]);
+    }
+
+    public function decreaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $min = null): bool
+    {
+        return $this->delegate('decreaseDocumentAttribute', [$collection, $id, $attribute, $value, $min]);
+    }
+
+    /**
+     * @throws Limit
+     * @throws DuplicateException
+     * @throws Exception
+     */
+    public function createUpgrades(): void
+    {
+        try {
+            parent::createCollection(
+                id: 'upgrades',
+                attributes: [
+                    new Document([
+                        '$id' => ID::custom('collectionId'),
+                        'type' => Database::VAR_STRING,
+                        'size' => Database::LENGTH_KEY,
+                        'required' => true,
+                        'signed' => true,
+                        'array' => false,
+                        'filters' => [],
+                        'default' => null,
+                        'format' => ''
+                    ]),
+                    new Document([
+                        '$id' => ID::custom('status'),
+                        'type' => Database::VAR_STRING,
+                        'size' => Database::LENGTH_KEY,
+                        'required' => false,
+                        'signed' => true,
+                        'array' => false,
+                        'filters' => [],
+                        'default' => null,
+                        'format' => ''
+                    ]),
+                ],
+                indexes: [
+                    new Document([
+                        '$id' => ID::custom('_unique_collection'),
+                        'type' => Database::INDEX_UNIQUE,
+                        'attributes' => ['collectionId'],
+                        'lengths' => [Database::LENGTH_KEY],
+                        'orders' => [],
+                    ]),
+                    new Document([
+                        '$id' => ID::custom('_status_index'),
+                        'type' => Database::INDEX_KEY,
+                        'attributes' => ['status'],
+                        'lengths' => [Database::LENGTH_KEY],
+                        'orders' => [Database::ORDER_ASC],
+                    ]),
+                ],
+            );
+        } catch (DuplicateException) {
+            // Ignore
+        }
+    }
+
+    /**
+     * @throws Exception
+     */
+    protected function getUpgradeStatus(string $collection): Document
+    {
+        if ($collection === 'upgrades' || $collection === Database::METADATA) {
+            return new Document([]);
+        }
+
+        return Authorization::skip(function () use ($collection) {
+            return $this->getDocument('upgrades', $collection);
+        });
+    }
+
+    protected function logError(string $action, \Throwable $err): void
+    {
+        foreach ($this->errorCallbacks as $callback) {
+            $callback($action, $err);
+        }
+    }
+}

From d2dc913fe70f14cfc621a7f6c105c2d1b9420681 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 6 Jun 2024 20:57:14 +1200
Subject: [PATCH 003/100] Add source property so filters call Database methods
 on source db instead of Mirror methods

---
 src/Database/Mirror.php           | 31 ++++++++++++++-------------
 src/Database/Mirroring/Filter.php | 35 ++++++++++++++++++++++++++++---
 2 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index db2fe58c1..c06019417 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -11,6 +11,7 @@
 
 class Mirror extends Database
 {
+    protected Database $source;
     protected Database $destination;
 
     /**
@@ -20,13 +21,6 @@ class Mirror extends Database
      */
     protected array $writeFilters = [];
 
-    /**
-     * Collections that should only be present in the source database
-     */
-    protected const SOURCE_ONLY_COLLECTIONS = [
-        'upgrades',
-    ];
-
     /**
      * Callbacks to run when an error occurs on the destination database
      *
@@ -34,6 +28,13 @@ class Mirror extends Database
      */
     protected array $errorCallbacks = [];
 
+    /**
+     * Collections that should only be present in the source database
+     */
+    protected const SOURCE_ONLY_COLLECTIONS = [
+        'upgrades',
+    ];
+
     /**
      * @param Adapter $adapter
      * @param Cache $cache
@@ -47,6 +48,7 @@ public function __construct(
         array $filters = [],
     ) {
         parent::__construct($adapter, $cache);
+        $this->source = new Database($adapter, $cache);
         $this->destination = $destination;
         $this->writeFilters = $filters;
     }
@@ -58,7 +60,7 @@ public function getDestination(): ?Database
 
     public function getSource(): Database
     {
-        return $this;
+        return $this->source;
     }
 
     /**
@@ -193,9 +195,9 @@ public function createDocument(string $collection, Document $document): Document
 
             foreach ($this->writeFilters as $filter) {
                 $clone = $filter->onCreateDocument(
-                    source: $this,
+                    source: $this->source,
                     destination: $this->destination,
-                    collection: $collection,
+                    collectionId: $collection,
                     document: $clone,
                 );
             }
@@ -224,9 +226,9 @@ public function updateDocument(string $collection, string $id, Document $documen
 
             foreach ($this->writeFilters as $filter) {
                 $clone = $filter->onUpdateDocument(
-                    source: $this,
+                    source: $this->source,
                     destination: $this->destination,
-                    collection: $collection,
+                    collectionId: $collection,
                     document: $clone,
                 );
             }
@@ -257,9 +259,10 @@ public function deleteDocument(string $collection, string $id): bool
 
             foreach ($this->writeFilters as $filter) {
                 $filter->onDeleteDocument(
-                    source: $this,
+                    source: $this->source,
                     destination: $this->destination,
-                    collection: $collection,
+                    collectionId: $collection,
+                    documentId: $id,
                 );
             }
         } catch (\Throwable $err) {
diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index 848dd7561..f247b9220 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -7,23 +7,52 @@
 
 abstract class Filter
 {
+    /**
+     * Called before document is created in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param Document $document
+     * @return Document
+     */
     abstract public function onCreateDocument(
         Database $source,
         Database $destination,
-        string $collection,
+        string $collectionId,
         Document $document,
     ): Document;
 
+
+    /**
+     * Called before document is updated in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param Document $document
+     * @return Document
+     */
     abstract public function onUpdateDocument(
         Database $source,
         Database $destination,
-        string $collection,
+        string $collectionId,
         Document $document,
     ): Document;
 
+    /**
+     * Called after document is deleted in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $documentId
+     * @return void
+     */
     abstract public function onDeleteDocument(
         Database $source,
         Database $destination,
-        string $collection,
+        string $collectionId,
+        string $documentId,
     ): void;
 }

From 2e02f0b8064ce2a36c1982a949696c5d82e7a853 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 6 Jun 2024 21:07:39 +1200
Subject: [PATCH 004/100] Use source instead of parent so changes applied via
 getSource are used

---
 src/Database/Database.php | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index dfafb8874..c331f5af7 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3507,11 +3507,13 @@ public function updateDocument(string $collection, string $id, Document $documen
         $old = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); // Skip ensures user does not need read permission for this
 
         $document = \array_merge($old->getArrayCopy(), $document->getArrayCopy());
+        $document['$createdAt'] = $old->getCreatedAt();                 // Make sure user doesn't switch createdAt
         $document['$collection'] = $old->getAttribute('$collection');   // Make sure user doesn't switch collection ID
+
         if($this->adapter->getSharedTables()) {
-            $document['$tenant'] = $old->getAttribute('$tenant');           // Make sure user doesn't switch tenant
+            $document['$tenant'] = $old->getAttribute('$tenant');       // Make sure user doesn't switch tenant
         }
-        $document['$createdAt'] = $old->getCreatedAt();                 // Make sure user doesn't switch createdAt
+
         $document = new Document($document);
 
         $collection = $this->silent(fn () => $this->getCollection($collection));

From ba9fc4e5ab4c5f33aed90ba402956074c2dc936a Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 6 Jun 2024 22:05:19 +1200
Subject: [PATCH 005/100] Add back nullable destination

---
 src/Database/Mirror.php | 77 ++++++++++++++++++++++++++---------------
 1 file changed, 50 insertions(+), 27 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index c06019417..005b58a5c 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -2,7 +2,6 @@
 
 namespace Utopia\Database;
 
-use Utopia\Cache\Cache;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Limit;
 use Utopia\Database\Helpers\ID;
@@ -12,7 +11,7 @@
 class Mirror extends Database
 {
     protected Database $source;
-    protected Database $destination;
+    protected ?Database $destination;
 
     /**
      * Filters to apply to documents before writing to the destination database
@@ -36,31 +35,32 @@ class Mirror extends Database
     ];
 
     /**
-     * @param Adapter $adapter
-     * @param Cache $cache
-     * @param Database $destination
+     * @param Database $source
+     * @param ?Database $destination
      * @param array<Filter> $filters
      */
     public function __construct(
-        Adapter $adapter,
-        Cache $cache,
-        Database $destination,
+        Database $source,
+        ?Database $destination = null,
         array $filters = [],
     ) {
-        parent::__construct($adapter, $cache);
-        $this->source = new Database($adapter, $cache);
+        parent::__construct(
+            $source->getAdapter(),
+            $source->getCache()
+        );
+        $this->source = $source;
         $this->destination = $destination;
         $this->writeFilters = $filters;
     }
 
-    public function getDestination(): ?Database
+    public function getSource(): Database
     {
-        return $this->destination;
+        return $this->source;
     }
 
-    public function getSource(): Database
+    public function getDestination(): ?Database
     {
-        return $this->source;
+        return $this->destination;
     }
 
     /**
@@ -79,7 +79,11 @@ public function onError(callable $callback): void
      */
     protected function delegate(string $method, array $args = []): mixed
     {
-        $result = parent::{$method}(...$args);
+        $result = $this->source->{$method}(...$args);
+
+        if ($this->destination === null) {
+            return $result;
+        }
 
         try {
             $result = $this->destination->{$method}(...$args);
@@ -112,7 +116,7 @@ public function create(?string $database = null): bool
 
     public function createCollection(string $id, array $attributes = [], array $indexes = [], array $permissions = null, bool $documentSecurity = true): Document
     {
-        $result = parent::createCollection(
+        $result = $this->source->createCollection(
             $id,
             $attributes,
             $indexes,
@@ -120,6 +124,10 @@ public function createCollection(string $id, array $attributes = [], array $inde
             $documentSecurity
         );
 
+        if ($this->destination === null) {
+            return $result;
+        }
+
         try {
             $this->destination->createCollection(
                 $id,
@@ -131,7 +139,7 @@ public function createCollection(string $id, array $attributes = [], array $inde
 
             $this->createUpgrades();
 
-            parent::createDocument('upgrades', new Document([
+            $this->source->createDocument('upgrades', new Document([
                 '$id' => $id,
                 'collectionId' => $id,
                 'status' => 'upgraded'
@@ -179,9 +187,12 @@ public function deleteIndex(string $collection, string $id): bool
 
     public function createDocument(string $collection, Document $document): Document
     {
-        $document = parent::createDocument($collection, $document);
+        $document = $this->source->createDocument($collection, $document);
 
-        if (\in_array($collection, self::SOURCE_ONLY_COLLECTIONS)) {
+        if (
+            \in_array($collection, self::SOURCE_ONLY_COLLECTIONS)
+            || $this->destination === null
+        ) {
             return $document;
         }
 
@@ -214,7 +225,14 @@ public function createDocument(string $collection, Document $document): Document
 
     public function updateDocument(string $collection, string $id, Document $document): Document
     {
-        $document = parent::updateDocument($collection, $id, $document);
+        $document = $this->source->updateDocument($collection, $id, $document);
+
+        if (
+            \in_array($collection, self::SOURCE_ONLY_COLLECTIONS)
+            || $this->destination === null
+        ) {
+            return $document;
+        }
 
         $upgrade = $this->getUpgradeStatus($collection);
         if ($upgrade->getAttribute('status', '') !== 'upgraded') {
@@ -233,11 +251,9 @@ public function updateDocument(string $collection, string $id, Document $documen
                 );
             }
 
-            if (!$this->destination->getDocument($collection, $id)->isEmpty()) {
-                $this->destination->setPreserveDates(true);
-                $this->destination->updateDocument($collection, $id, $clone);
-                $this->destination->setPreserveDates(false);
-            }
+            $this->destination->setPreserveDates(true);
+            $this->destination->updateDocument($collection, $id, $clone);
+            $this->destination->setPreserveDates(false);
         } catch (\Throwable $err) {
             $this->logError('updateDocument', $err);
         }
@@ -247,7 +263,14 @@ public function updateDocument(string $collection, string $id, Document $documen
 
     public function deleteDocument(string $collection, string $id): bool
     {
-        $result = parent::deleteDocument($collection, $id);
+        $result = $this->source->deleteDocument($collection, $id);
+
+        if (
+            \in_array($collection, self::SOURCE_ONLY_COLLECTIONS)
+            || $this->destination === null
+        ) {
+            return $result;
+        }
 
         $upgrade = $this->getUpgradeStatus($collection);
         if ($upgrade->getAttribute('status', '') !== 'upgraded') {
@@ -354,7 +377,7 @@ public function decreaseDocumentAttribute(string $collection, string $id, string
     public function createUpgrades(): void
     {
         try {
-            parent::createCollection(
+            $this->source->createCollection(
                 id: 'upgrades',
                 attributes: [
                     new Document([

From c409fc51a19ce5d9cb5af3accad1f4ff150f63df Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 6 Jun 2024 22:09:36 +1200
Subject: [PATCH 006/100] Add base tests

---
 src/Database/Mirror.php          |  10 ++
 tests/e2e/Adapter/MirrorTest.php | 156 +++++++++++++++++++++++++++++++
 2 files changed, 166 insertions(+)
 create mode 100644 tests/e2e/Adapter/MirrorTest.php

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 005b58a5c..a19ef1528 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -94,6 +94,16 @@ protected function delegate(string $method, array $args = []): mixed
         return $result;
     }
 
+    public function setDatabase(string $name): Database
+    {
+        return $this->delegate('setDatabase', [$name]);
+    }
+
+    public function setNamespace(string $namespace): Database
+    {
+        return $this->delegate('setNamespace', [$namespace]);
+    }
+
     public function enableValidation(): self
     {
         return $this->delegate('enableValidation');
diff --git a/tests/e2e/Adapter/MirrorTest.php b/tests/e2e/Adapter/MirrorTest.php
new file mode 100644
index 000000000..258241945
--- /dev/null
+++ b/tests/e2e/Adapter/MirrorTest.php
@@ -0,0 +1,156 @@
+<?php
+
+namespace Tests\E2E\Adapter;
+
+use PDO;
+use Redis;
+use Utopia\Cache\Adapter\Redis as RedisAdapter;
+use Utopia\Cache\Cache;
+use Utopia\Database\Adapter\MariaDB;
+use Utopia\Database\Database;
+use Utopia\Database\Exception;
+use Utopia\Database\Exception\Conflict;
+use Utopia\Database\Exception\Duplicate;
+use Utopia\Database\Exception\Limit;
+use Utopia\Database\Helpers\Permission;
+use Utopia\Database\Helpers\Role;
+use Utopia\Database\Mirror;
+
+class MirrorTest extends Base
+{
+    protected static ?Mirror $database = null;
+    protected static Database $source;
+    protected static Database $destination;
+
+    protected static string $namespace;
+
+    /**
+     * @throws \RedisException
+     * @throws Exception
+     */
+    protected static function getDatabase(bool $fresh = false): Mirror
+    {
+        if (!is_null(self::$database) && !$fresh) {
+            return self::$database;
+        }
+
+        $dbHost = 'mariadb';
+        $dbPort = '3306';
+        $dbUser = 'root';
+        $dbPass = 'password';
+
+        $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes());
+        $redis = new Redis();
+        $redis->connect('redis');
+        $redis->flushAll();
+        $cache = new Cache(new RedisAdapter($redis));
+
+        self::$source = new Database(new MariaDB($pdo), $cache);
+
+        $mirrorHost = 'mariadb-mirror';
+        $mirrorPort = '3306';
+        $mirrorUser = 'root';
+        $mirrorPass = 'password';
+
+        $mirrorPdo = new PDO("mysql:host={$mirrorHost};port={$mirrorPort};charset=utf8mb4", $mirrorUser, $mirrorPass, MariaDB::getPDOAttributes());
+
+        self::$destination = new Database(new MariaDB($mirrorPdo), $cache);
+
+        $database = new Mirror(self::$source, self::$destination);
+
+        $database
+            ->setDatabase('utopiaTests')
+            ->setNamespace(static::$namespace = 'myapp_' . uniqid());
+
+        if ($database->exists()) {
+            $database->delete();
+        }
+
+        $database->create();
+
+        return self::$database = $database;
+    }
+
+    protected static function getAdapterName(): string
+    {
+        return "Mirror";
+    }
+
+    /**
+     * @throws Exception
+     * @throws \RedisException
+     */
+    public function testGetSource(): void
+    {
+        $database = self::getDatabase();
+        $source = $database->getSource();
+        $this->assertInstanceOf(Database::class, $source);
+        $this->assertEquals(self::$source, $source);
+    }
+
+    /**
+     * @throws Exception
+     * @throws \RedisException
+     */
+    public function testGetDestination(): void
+    {
+        $database = self::getDatabase();
+        $destination = $database->getDestination();
+        $this->assertInstanceOf(Database::class, $destination);
+        $this->assertEquals(self::$destination, $destination);
+    }
+
+    /**
+     * @throws Limit
+     * @throws Duplicate
+     * @throws Exception
+     * @throws \RedisException
+     */
+    public function testCreateCollection(): void
+    {
+        $database = self::getDatabase();
+
+        $database->createCollection('testCreateCollection');
+
+        // Assert collection exists in both databases
+        $this->assertFalse($database->getSource()->getCollection('testCreateCollection')->isEmpty());
+        $this->assertFalse($database->getDestination()->getCollection('testCreateCollection')->isEmpty());
+    }
+
+    /**
+     * @throws Limit
+     * @throws Duplicate
+     * @throws \RedisException
+     * @throws Conflict
+     * @throws Exception
+     */
+    public function testUpdateCollection(): void
+    {
+        $database = self::getDatabase();
+
+        $database->createCollection('testUpdateCollection', permissions: [
+            Permission::read(Role::any()),
+        ]);
+
+        $collection = $database->getCollection('testUpdateCollection');
+
+        $database->updateCollection(
+            'testUpdateCollection',
+            [
+                Permission::read(Role::users()),
+            ],
+            $collection->getAttribute('documentSecurity')
+        );
+
+        // Asset both databases have updated the collection
+        $this->assertEquals(
+            [Permission::read(Role::users())],
+            $database->getSource()->getCollection('testUpdateCollection')->getPermissions()
+        );
+
+        $this->assertEquals(
+            [Permission::read(Role::users())],
+            $database->getDestination()->getCollection('testUpdateCollection')->getPermissions()
+        );
+    }
+}

From 712f3e8f6264c678c9314edeba776b9875b4ec5d Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 17:36:55 +1200
Subject: [PATCH 007/100] Add xdebug

---
 Dockerfile         | 47 +++++++++++++++++++++++++++++++++++-----------
 composer.lock      |  2 +-
 dev/xdebug.ini     |  8 ++++++++
 docker-compose.yml |  5 +++--
 4 files changed, 48 insertions(+), 14 deletions(-)
 create mode 100644 dev/xdebug.ini

diff --git a/Dockerfile b/Dockerfile
index 443bc3d13..7c46fd8f9 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,27 +1,42 @@
 FROM composer:2.0 as composer
 
-ARG TESTING=false
-ENV TESTING=$TESTING
+ARG DEBUG=false
+ENV DEBUG=$DEBUG
 
 WORKDIR /usr/local/src/
 
 COPY composer.lock /usr/local/src/
 COPY composer.json /usr/local/src/
 
-RUN composer install --ignore-platform-reqs --optimize-autoloader \
-    --no-plugins --no-scripts --prefer-dist
+RUN composer install \
+    --ignore-platform-reqs \
+    --optimize-autoloader \
+    --no-plugins \
+    --no-scripts \
+    --prefer-dist
     
-FROM php:8.3.3-cli-alpine3.19 as compile
+FROM php:8.3.7-cli-alpine3.19 as compile
+
+ENV PHP_REDIS_VERSION="6.0.2" \
+    PHP_SWOOLE_VERSION="v5.1.3" \
+    PHP_MONGO_VERSION="1.16.1" \
+    PHP_XDEBUG_VERSION="3.3.2"
 
-ENV PHP_REDIS_VERSION=6.0.2 \
-    PHP_SWOOLE_VERSION=v5.1.2 \
-    PHP_MONGO_VERSION=1.16.1
-    
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 RUN \
   apk update \
-  && apk add --no-cache postgresql-libs postgresql-dev make automake autoconf gcc g++ git brotli-dev \
+  && apk add --no-cache \
+    postgresql-libs \
+    postgresql-dev \
+    make \
+    automake \
+    autoconf \
+    gcc \
+    g++ \
+    git \
+    brotli-dev \
+    linux-headers \
   && docker-php-ext-install opcache pgsql pdo_mysql pdo_pgsql \
   && apk del postgresql-dev \
   && rm -rf /var/cache/apk/*
@@ -63,6 +78,15 @@ RUN \
    && ./configure --enable-pcov \
    && make && make install
 
+## XDebug Extension
+FROM compile AS xdebug
+RUN \
+  git clone --depth 1 --branch $PHP_XDEBUG_VERSION https://github.com/xdebug/xdebug && \
+  cd xdebug && \
+  phpize && \
+  ./configure && \
+  make && make install
+
 FROM compile as final
 
 LABEL maintainer="team@appwrite.io"
@@ -73,6 +97,7 @@ RUN echo extension=redis.so >> /usr/local/etc/php/conf.d/redis.ini
 RUN echo extension=swoole.so >> /usr/local/etc/php/conf.d/swoole.ini
 RUN echo extension=mongodb.so >> /usr/local/etc/php/conf.d/mongodb.ini
 RUN echo extension=pcov.so >> /usr/local/etc/php/conf.d/pcov.ini
+RUN echo extension=xdebug.so >> /usr/local/etc/php/conf.d/xdebug.ini
 
 RUN mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini"
 
@@ -85,8 +110,8 @@ COPY --from=swoole /usr/local/lib/php/extensions/no-debug-non-zts-20230831/swool
 COPY --from=redis /usr/local/lib/php/extensions/no-debug-non-zts-20230831/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/
 COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20230831/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/
 COPY --from=pcov /usr/local/lib/php/extensions/no-debug-non-zts-20230831/pcov.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/
+COPY --from=xdebug /usr/local/lib/php/extensions/no-debug-non-zts-20230831/xdebug.so /usr/local/lib/php/extensions/no-debug-non-zts-20230831/
 
-# Add Source Code
 COPY ./bin /usr/src/code/bin
 COPY ./src /usr/src/code/src
 
diff --git a/composer.lock b/composer.lock
index 37d72aa6d..53540fd01 100644
--- a/composer.lock
+++ b/composer.lock
@@ -2603,5 +2603,5 @@
         "php": ">=8.0"
     },
     "platform-dev": [],
-    "plugin-api-version": "2.2.0"
+    "plugin-api-version": "2.6.0"
 }
diff --git a/dev/xdebug.ini b/dev/xdebug.ini
new file mode 100644
index 000000000..0f3241d37
--- /dev/null
+++ b/dev/xdebug.ini
@@ -0,0 +1,8 @@
+zend_extension = xdebug.so
+
+[xdebug]
+xdebug.mode = develop,debug
+xdebug.start_with_request = yes
+xdebug.client_host=host.docker.internal
+xdebug.client_port = 9003
+xdebug.log = /tmp/xdebug.log
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index c5a9128f7..4d10e8cb6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,6 +4,8 @@ services:
     image: databases-dev
     build:
       context: .
+      args:
+        - DEBUG=true
     networks:
       - database
     volumes:
@@ -11,8 +13,7 @@ services:
       - ./src:/usr/src/code/src
       - ./tests:/usr/src/code/tests
       - ./phpunit.xml:/usr/src/code/phpunit.xml
-    ports:
-      - "8708:8708"
+      - ./dev/xdebug.ini:/usr/local/etc/php/conf.d/xdebug.ini
 
   adminer:
     image: adminer

From f812b84203f2a29fd5891f92649ba68d1922147a Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 18:30:45 +1200
Subject: [PATCH 008/100] Fix getting upgrade status

---
 src/Database/Mirror.php | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index a19ef1528..448018ee9 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -438,14 +438,18 @@ public function createUpgrades(): void
     /**
      * @throws Exception
      */
-    protected function getUpgradeStatus(string $collection): Document
+    protected function getUpgradeStatus(string $collection): ?Document
     {
         if ($collection === 'upgrades' || $collection === Database::METADATA) {
-            return new Document([]);
+            return new Document();
         }
 
         return Authorization::skip(function () use ($collection) {
-            return $this->getDocument('upgrades', $collection);
+            try {
+                return $this->getDocument('upgrades', $collection);
+            } catch (\Throwable) {
+                return null;
+            }
         });
     }
 

From 757fa9113e69ad7f3661113bb85503ed9d0db15b Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 18:31:43 +1200
Subject: [PATCH 009/100] Add missing create/update batch methods

---
 src/Database/Mirror.php | 94 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 448018ee9..f68291c99 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -233,6 +233,53 @@ public function createDocument(string $collection, Document $document): Document
         return $document;
     }
 
+    public function createDocuments(
+        string $collection,
+        array $documents,
+        int $batchSize = self::INSERT_BATCH_SIZE
+    ): array {
+        $documents = $this->source->createDocuments($collection, $documents, $batchSize);
+
+        if (
+            \in_array($collection, self::SOURCE_ONLY_COLLECTIONS)
+            || $this->destination === null
+        ) {
+            return $documents;
+        }
+
+        $upgrade = $this->getUpgradeStatus($collection);
+        if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
+            return $documents;
+        }
+
+        try {
+            $clones = [];
+
+            foreach ($documents as $document) {
+                $clone = clone $document;
+
+                foreach ($this->writeFilters as $filter) {
+                    $clone = $filter->onCreateDocument(
+                        source: $this->source,
+                        destination: $this->destination,
+                        collectionId: $collection,
+                        document: $clone,
+                    );
+                }
+
+                $clones[] = $clone;
+            }
+
+            $this->destination->setPreserveDates(true);
+            $this->destination->createDocuments($collection, $clones, $batchSize);
+            $this->destination->setPreserveDates(false);
+        } catch (\Throwable $err) {
+            $this->logError('createDocuments', $err);
+        }
+
+        return $documents;
+    }
+
     public function updateDocument(string $collection, string $id, Document $document): Document
     {
         $document = $this->source->updateDocument($collection, $id, $document);
@@ -271,6 +318,53 @@ public function updateDocument(string $collection, string $id, Document $documen
         return $document;
     }
 
+    public function updateDocuments(
+        string $collection,
+        array $documents,
+        int $batchSize = self::INSERT_BATCH_SIZE
+    ): array {
+        $documents = $this->source->updateDocuments($collection, $documents, $batchSize);
+
+        if (
+            \in_array($collection, self::SOURCE_ONLY_COLLECTIONS)
+            || $this->destination === null
+        ) {
+            return $documents;
+        }
+
+        $upgrade = $this->getUpgradeStatus($collection);
+        if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
+            return $documents;
+        }
+
+        try {
+            $clones = [];
+
+            foreach ($documents as $document) {
+                $clone = clone $document;
+
+                foreach ($this->writeFilters as $filter) {
+                    $clone = $filter->onUpdateDocument(
+                        source: $this->source,
+                        destination: $this->destination,
+                        collectionId: $collection,
+                        document: $clone,
+                    );
+                }
+
+                $clones[] = $clone;
+            }
+
+            $this->destination->setPreserveDates(true);
+            $this->destination->updateDocuments($collection, $clones, $batchSize);
+            $this->destination->setPreserveDates(false);
+        } catch (\Throwable $err) {
+            $this->logError('updateDocuments', $err);
+        }
+
+        return $documents;
+    }
+
     public function deleteDocument(string $collection, string $id): bool
     {
         $result = $this->source->deleteDocument($collection, $id);

From 5af4e8e862095d6979d3136061d834599346565d Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 18:32:19 +1200
Subject: [PATCH 010/100] Fix upgrade checks

---
 src/Database/Mirror.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index f68291c99..09d070d8d 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -207,7 +207,7 @@ public function createDocument(string $collection, Document $document): Document
         }
 
         $upgrade = $this->getUpgradeStatus($collection);
-        if ($upgrade->getAttribute('status', '') !== 'upgraded') {
+        if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $document;
         }
 
@@ -292,7 +292,7 @@ public function updateDocument(string $collection, string $id, Document $documen
         }
 
         $upgrade = $this->getUpgradeStatus($collection);
-        if ($upgrade->getAttribute('status', '') !== 'upgraded') {
+        if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $document;
         }
 
@@ -377,7 +377,7 @@ public function deleteDocument(string $collection, string $id): bool
         }
 
         $upgrade = $this->getUpgradeStatus($collection);
-        if ($upgrade->getAttribute('status', '') !== 'upgraded') {
+        if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $result;
         }
 

From 995be2e26437705259ce2caf431ddc1b213992ad Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 18:36:09 +1200
Subject: [PATCH 011/100] Delegate more setters and delete

---
 src/Database/Mirror.php | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 09d070d8d..1564fd277 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -104,7 +104,22 @@ public function setNamespace(string $namespace): Database
         return $this->delegate('setNamespace', [$namespace]);
     }
 
-    public function enableValidation(): self
+    public function setSharedTables(bool $sharedTables): static
+    {
+        return $this->delegate('setSharedTables', [$sharedTables]);
+    }
+
+    public function setTenant(?int $tenant): static
+    {
+        return $this->delegate('setTenant', [$tenant]);
+    }
+
+    public function setPreserveDates(bool $preserve): static
+    {
+        return $this->delegate('setPreserveDates', [$preserve]);
+    }
+
+    public function enableValidation(): static
     {
         return $this->delegate('enableValidation');
     }
@@ -114,9 +129,9 @@ public function disableValidation(): self
         return $this->delegate('disableValidation');
     }
 
-    public function delete(?string $database = null): bool
+    public function exists(?string $database = null, ?string $collection = null): bool
     {
-        return $this->delegate('delete', [$database]);
+        return $this->delegate('exists', [$database, $collection]);
     }
 
     public function create(?string $database = null): bool
@@ -124,6 +139,11 @@ public function create(?string $database = null): bool
         return $this->delegate('create', [$database]);
     }
 
+    public function delete(?string $database = null): bool
+    {
+        return $this->delegate('delete', [$database]);
+    }
+
     public function createCollection(string $id, array $attributes = [], array $indexes = [], array $permissions = null, bool $documentSecurity = true): Document
     {
         $result = $this->source->createCollection(

From b5fae84d9f669468b26507318653a4680325d747 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 18:51:23 +1200
Subject: [PATCH 012/100] Static returns

---
 src/Database/Adapter.php            | 14 +++++-----
 src/Database/Database.php           | 42 ++++++++++++++---------------
 src/Database/Document.php           | 13 ++++-----
 src/Database/Helpers/Permission.php |  2 +-
 src/Database/Helpers/Role.php       | 14 +++++-----
 src/Database/Mirror.php             |  8 +++---
 tests/e2e/Adapter/Base.php          |  6 ++---
 7 files changed, 49 insertions(+), 50 deletions(-)

diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php
index ee138b7e0..586040828 100644
--- a/src/Database/Adapter.php
+++ b/src/Database/Adapter.php
@@ -38,7 +38,7 @@ abstract class Adapter
      *
      * @return $this
      */
-    public function setDebug(string $key, mixed $value): self
+    public function setDebug(string $key, mixed $value): static
     {
         $this->debug[$key] = $value;
 
@@ -54,9 +54,9 @@ public function getDebug(): array
     }
 
     /**
-     * @return self
+     * @return static
      */
-    public function resetDebug(): self
+    public function resetDebug(): static
     {
         $this->debug = [];
 
@@ -192,7 +192,7 @@ public function getTenant(): ?int
      * @param mixed $value
      * @return $this
      */
-    public function setMetadata(string $key, mixed $value): self
+    public function setMetadata(string $key, mixed $value): static
     {
         $this->metadata[$key] = $value;
 
@@ -223,7 +223,7 @@ public function getMetadata(): array
      *
      * @return $this
      */
-    public function resetMetadata(): self
+    public function resetMetadata(): static
     {
         $this->metadata = [];
 
@@ -236,9 +236,9 @@ public function resetMetadata(): self
      * @param string $event
      * @param string $name
      * @param ?callable $callback
-     * @return self
+     * @return static
      */
-    public function before(string $event, string $name = '', ?callable $callback = null): self
+    public function before(string $event, string $name = '', ?callable $callback = null): static
     {
         if (!isset($this->transformations[$event])) {
             $this->transformations[$event] = [];
diff --git a/src/Database/Database.php b/src/Database/Database.php
index faa316cc8..868131d07 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -430,9 +430,9 @@ function (?string $value) {
      * @param string $event
      * @param string $name
      * @param callable $callback
-     * @return self
+     * @return static
      */
-    public function on(string $event, string $name, callable $callback): self
+    public function on(string $event, string $name, callable $callback): static
     {
         if (!isset($this->listeners[$event])) {
             $this->listeners[$event] = [];
@@ -450,7 +450,7 @@ public function on(string $event, string $name, callable $callback): self
      * @param callable $callback
      * @return $this
      */
-    public function before(string $event, string $name, callable $callback): self
+    public function before(string $event, string $name, callable $callback): static
     {
         $this->adapter->before($event, $name, $callback);
 
@@ -563,7 +563,7 @@ public function withRequestTimestamp(?\DateTime $requestTimestamp, callable $cal
      *
      * @throws DatabaseException
      */
-    public function setNamespace(string $namespace): self
+    public function setNamespace(string $namespace): static
     {
         $this->adapter->setNamespace($namespace);
 
@@ -587,10 +587,10 @@ public function getNamespace(): string
      *
      * @param string $name
      *
-     * @return self
+     * @return static
      * @throws DatabaseException
      */
-    public function setDatabase(string $name): self
+    public function setDatabase(string $name): static
     {
         $this->adapter->setDatabase($name);
 
@@ -617,7 +617,7 @@ public function getDatabase(): string
      *
      * @return $this
      */
-    public function setCache(Cache $cache): self
+    public function setCache(Cache $cache): static
     {
         $this->cache = $cache;
         return $this;
@@ -639,7 +639,7 @@ public function getCache(): Cache
      * @param string $name
      * @return $this
      */
-    public function setCacheName(string $name): self
+    public function setCacheName(string $name): static
     {
         $this->cacheName = $name;
 
@@ -661,9 +661,9 @@ public function getCacheName(): string
      *
      * @param string $key
      * @param mixed $value
-     * @return self
+     * @return static
      */
-    public function setMetadata(string $key, mixed $value): self
+    public function setMetadata(string $key, mixed $value): static
     {
         $this->adapter->setMetadata($key, $value);
 
@@ -695,10 +695,10 @@ public function resetMetadata(): void
      *
      * @param int $milliseconds
      * @param string $event
-     * @return self
+     * @return static
      * @throws Exception
      */
-    public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): self
+    public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): static
     {
         $this->adapter->setTimeout($milliseconds, $event);
 
@@ -721,7 +721,7 @@ public function clearTimeout(string $event = Database::EVENT_ALL): void
      *
      * @return $this
      */
-    public function enableFilters(): self
+    public function enableFilters(): static
     {
         $this->filter = true;
 
@@ -733,7 +733,7 @@ public function enableFilters(): self
      *
      * @return $this
      */
-    public function disableFilters(): self
+    public function disableFilters(): static
     {
         $this->filter = false;
 
@@ -755,7 +755,7 @@ public function getInstanceFilters(): array
      *
      * @return $this
      */
-    public function enableValidation(): self
+    public function enableValidation(): static
     {
         $this->validate = true;
 
@@ -767,7 +767,7 @@ public function enableValidation(): self
      *
      * @return $this
      */
-    public function disableValidation(): self
+    public function disableValidation(): static
     {
         $this->validate = false;
 
@@ -780,9 +780,9 @@ public function disableValidation(): self
      * Set whether to share tables between tenants
      *
      * @param bool $sharedTables
-     * @return self
+     * @return static
      */
-    public function setSharedTables(bool $sharedTables): self
+    public function setSharedTables(bool $sharedTables): static
     {
         $this->adapter->setSharedTables($sharedTables);
 
@@ -795,16 +795,16 @@ public function setSharedTables(bool $sharedTables): self
      * Set tenant to use if tables are shared
      *
      * @param ?int $tenant
-     * @return self
+     * @return static
      */
-    public function setTenant(?int $tenant): self
+    public function setTenant(?int $tenant): static
     {
         $this->adapter->setTenant($tenant);
 
         return $this;
     }
 
-    public function setPreserveDates(bool $preserve): self
+    public function setPreserveDates(bool $preserve): static
     {
         $this->preserveDates = $preserve;
 
diff --git a/src/Database/Document.php b/src/Database/Document.php
index 7afddc25f..f605f615f 100644
--- a/src/Database/Document.php
+++ b/src/Database/Document.php
@@ -210,9 +210,9 @@ public function getAttribute(string $name, mixed $default = null): mixed
      * @param mixed $value
      * @param string $type
      *
-     * @return self
+     * @return static
      */
-    public function setAttribute(string $key, mixed $value, string $type = self::SET_TYPE_ASSIGN): self
+    public function setAttribute(string $key, mixed $value, string $type = self::SET_TYPE_ASSIGN): static
     {
         switch ($type) {
             case self::SET_TYPE_ASSIGN:
@@ -235,9 +235,9 @@ public function setAttribute(string $key, mixed $value, string $type = self::SET
      * Set Attributes.
      *
      * @param array<string, mixed> $attributes
-     * @return self
+     * @return static
      */
-    public function setAttributes(array $attributes): self
+    public function setAttributes(array $attributes): static
     {
         foreach ($attributes as $key => $value) {
             $this->setAttribute($key, $value);
@@ -253,14 +253,15 @@ public function setAttributes(array $attributes): self
      *
      * @param string $key
      *
-     * @return self
+     * @return static
      */
-    public function removeAttribute(string $key): self
+    public function removeAttribute(string $key): static
     {
         if (\array_key_exists($key, (array)$this)) {
             unset($this[$key]);
         }
 
+        /* @phpstan-ignore-next-line */
         return $this;
     }
 
diff --git a/src/Database/Helpers/Permission.php b/src/Database/Helpers/Permission.php
index df0f1a5c0..18c4fe5a9 100644
--- a/src/Database/Helpers/Permission.php
+++ b/src/Database/Helpers/Permission.php
@@ -77,7 +77,7 @@ public function getDimension(): string
      * Parse a permission string into a Permission object
      *
      * @param string $permission
-     * @return Permission
+     * @return self
      * @throws Exception
      */
     public static function parse(string $permission): self
diff --git a/src/Database/Helpers/Role.php b/src/Database/Helpers/Role.php
index aaf755038..1682cb547 100644
--- a/src/Database/Helpers/Role.php
+++ b/src/Database/Helpers/Role.php
@@ -56,7 +56,7 @@ public function getDimension(): string
      * Parse a role string into a Role object
      *
      * @param string $role
-     * @return Role
+     * @return self
      * @throws \Exception
      */
     public static function parse(string $role): self
@@ -110,7 +110,7 @@ public static function parse(string $role): self
      *
      * @param string $identifier
      * @param string $status
-     * @return Role
+     * @return self
      */
     public static function user(string $identifier, string $status = ''): Role
     {
@@ -121,7 +121,7 @@ public static function user(string $identifier, string $status = ''): Role
      * Create a users role
      *
      * @param string $status
-     * @return Role
+     * @return self
      */
     public static function users(string $status = ''): self
     {
@@ -133,7 +133,7 @@ public static function users(string $status = ''): self
      *
      * @param string $identifier
      * @param string $dimension
-     * @return Role
+     * @return self
      */
     public static function team(string $identifier, string $dimension = ''): self
     {
@@ -144,7 +144,7 @@ public static function team(string $identifier, string $dimension = ''): self
      * Create a label role from the given ID
      *
      * @param string $identifier
-     * @return Role
+     * @return self
      */
     public static function label(string $identifier): self
     {
@@ -154,7 +154,7 @@ public static function label(string $identifier): self
     /**
      * Create an any satisfy role
      *
-     * @return Role
+     * @return self
      */
     public static function any(): Role
     {
@@ -164,7 +164,7 @@ public static function any(): Role
     /**
      * Create a guests role
      *
-     * @return Role
+     * @return self
      */
     public static function guests(): self
     {
diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 1564fd277..659b8fc54 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -94,12 +94,12 @@ protected function delegate(string $method, array $args = []): mixed
         return $result;
     }
 
-    public function setDatabase(string $name): Database
+    public function setDatabase(string $name): static
     {
         return $this->delegate('setDatabase', [$name]);
     }
 
-    public function setNamespace(string $namespace): Database
+    public function setNamespace(string $namespace): static
     {
         return $this->delegate('setNamespace', [$namespace]);
     }
@@ -124,7 +124,7 @@ public function enableValidation(): static
         return $this->delegate('enableValidation');
     }
 
-    public function disableValidation(): self
+    public function disableValidation(): static
     {
         return $this->delegate('disableValidation');
     }
@@ -562,7 +562,7 @@ protected function getUpgradeStatus(string $collection): ?Document
             try {
                 return $this->getDocument('upgrades', $collection);
             } catch (\Throwable) {
-                return null;
+                return;
             }
         });
     }
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 87c461f78..b154539aa 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -75,7 +75,6 @@ public function testCreateExistsDelete(): void
         $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase));
         $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase));
         $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase));
-        $this->assertEquals(static::getDatabase(), static::getDatabase()->setDatabase($this->testDatabase));
         $this->assertEquals(true, static::getDatabase()->create());
     }
 
@@ -317,7 +316,6 @@ public function testVirtualRelationsAttributes(): void
         /**
          * Success for later test update
          */
-
         $doc = static::getDatabase()->createDocument('v1', new Document([
             '$id' => 'man',
             '$permissions' => [
@@ -744,7 +742,8 @@ public function testPreserveDatesUpdate(): void
         $newDate = '2000-01-01T10:00:00.000+00:00';
 
         $doc1->setAttribute('$updatedAt', $newDate);
-        static::getDatabase()->updateDocument('preserve_update_dates', 'doc1', $doc1);
+        $doc1 = static::getDatabase()->updateDocument('preserve_update_dates', 'doc1', $doc1);
+        $this->assertEquals($newDate, $doc1->getAttribute('$updatedAt'));
         $doc1 = static::getDatabase()->getDocument('preserve_update_dates', 'doc1');
         $this->assertEquals($newDate, $doc1->getAttribute('$updatedAt'));
 
@@ -14983,7 +14982,6 @@ public function testIsolationModes(): void
         /**
          * Table
          */
-
         $tenant1 = 1;
         $tenant2 = 2;
 

From 81516d51675cf0a5c5f75cd539beca32c16e6489 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 18:51:44 +1200
Subject: [PATCH 013/100] Add mirror containers to compose stack

---
 docker-compose.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index 4d10e8cb6..90f781c7a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -44,6 +44,16 @@ services:
       - "8701:3306"
     environment:
       - MYSQL_ROOT_PASSWORD=password
+
+  mariadb-mirror:
+    image: mariadb:10.11
+    container_name: utopia-mariadb-mirror
+    networks:
+      - database
+    ports:
+      - "8704:3306"
+    environment:
+      - MYSQL_ROOT_PASSWORD=password
   
   mongo:
     image: mongo:5.0
@@ -72,6 +82,22 @@ services:
     cap_add:
       - SYS_NICE
 
+  mysql-mirror:
+    image: mysql:8.0.33
+    container_name: utopia-mysql-mirror
+    networks:
+      - database
+    ports:
+      - "8705:3307"
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_DATABASE: default
+      MYSQL_USER: user
+      MYSQL_PASSWORD: password
+      MYSQL_TCP_PORT: 3307
+    cap_add:
+      - SYS_NICE
+
   redis:
     image: redis:6.0-alpine
     container_name: utopia-redis

From f181c7efebf18a1d43f0717d20ca38383dc0ff43 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 19:45:07 +1200
Subject: [PATCH 014/100] Fix return type for setters

---
 src/Database/Database.php |  2 +-
 src/Database/Mirror.php   | 28 +++++++++++++++++++++-------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 868131d07..c56d811ef 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -1478,7 +1478,7 @@ protected function updateAttributeMeta(string $collection, string $id, callable
         $updateCallback($attributes[$index], $collection, $index);
 
         // Save
-        $collection->setAttribute('attributes', $attributes, Document::SET_TYPE_ASSIGN);
+        $collection->setAttribute('attributes', $attributes);
 
         $this->silent(fn () => $this->updateDocument(self::METADATA, $collection->getId(), $collection));
 
diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 659b8fc54..79aa02442 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -96,37 +96,51 @@ protected function delegate(string $method, array $args = []): mixed
 
     public function setDatabase(string $name): static
     {
-        return $this->delegate('setDatabase', [$name]);
+        $this->delegate('setDatabase', [$name]);
+
+        return $this;
     }
 
     public function setNamespace(string $namespace): static
     {
-        return $this->delegate('setNamespace', [$namespace]);
+        $this->delegate('setNamespace', [$namespace]);
+
+        return $this;
     }
 
     public function setSharedTables(bool $sharedTables): static
     {
-        return $this->delegate('setSharedTables', [$sharedTables]);
+        $this->delegate('setSharedTables', [$sharedTables]);
+
+        return $this;
     }
 
     public function setTenant(?int $tenant): static
     {
-        return $this->delegate('setTenant', [$tenant]);
+        $this->delegate('setTenant', [$tenant]);
+
+        return $this;
     }
 
     public function setPreserveDates(bool $preserve): static
     {
-        return $this->delegate('setPreserveDates', [$preserve]);
+        $this->delegate('setPreserveDates', [$preserve]);
+
+        return $this;
     }
 
     public function enableValidation(): static
     {
-        return $this->delegate('enableValidation');
+        $this->delegate('enableValidation');
+
+        return $this;
     }
 
     public function disableValidation(): static
     {
-        return $this->delegate('disableValidation');
+        $this->delegate('disableValidation');
+
+        return $this;
     }
 
     public function exists(?string $database = null, ?string $collection = null): bool

From a4c41095ba7b8659c8924f7382a0e6a62fc15a51 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 19:46:09 +1200
Subject: [PATCH 015/100] Fix events firing for destination db

---
 src/Database/Mirror.php | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 79aa02442..cba3a3319 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -143,6 +143,23 @@ public function disableValidation(): static
         return $this;
     }
 
+    public function on(string $event, string $name, callable $callback): static
+    {
+        $this->source->on($event, $name, $callback);
+
+        return $this;
+    }
+
+    protected function trigger(string $event, mixed $args = null): void
+    {
+        $this->source->trigger($event, $args);
+    }
+
+    public function silent(callable $callback, array $listeners = null): mixed
+    {
+        return $this->source->silent($callback, $listeners);
+    }
+
     public function exists(?string $database = null, ?string $collection = null): bool
     {
         return $this->delegate('exists', [$database, $collection]);

From 944aed63151c97351f4ca9dfd12573b0d408b342 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 19:46:54 +1200
Subject: [PATCH 016/100] Fix timestamp conflicts

---
 src/Database/Mirror.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index cba3a3319..be350d296 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -160,6 +160,11 @@ public function silent(callable $callback, array $listeners = null): mixed
         return $this->source->silent($callback, $listeners);
     }
 
+    public function withRequestTimestamp(?\DateTime $requestTimestamp, callable $callback): mixed
+    {
+        return $this->delegate('withRequestTimestamp', [$requestTimestamp, $callback]);
+    }
+
     public function exists(?string $database = null, ?string $collection = null): bool
     {
         return $this->delegate('exists', [$database, $collection]);

From fd22e9a9346008f94937cf1a3c3bc454b5df5a35 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 20:27:51 +1200
Subject: [PATCH 017/100] Fix setters on parents

---
 src/Database/Adapter/MariaDB.php | 3 +--
 src/Database/Database.php        | 2 ++
 src/Database/Mirror.php          | 6 ++++++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 4086b90f9..369cadfa1 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -841,8 +841,7 @@ public function createDocument(string $collection, Document $document): Document
                     $stmtPermissions->bindValue(':_tenant', $this->tenant);
                 }
             }
-
-
+            
             $stmt->execute();
 
             $document['$internalId'] = $this->getDocument($collection, $document->getId(), [Query::select(['$internalId'])])->getInternalId();
diff --git a/src/Database/Database.php b/src/Database/Database.php
index c56d811ef..35f446c9a 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3109,9 +3109,11 @@ public function createDocument(string $collection, Document $document): Document
         if ($this->resolveRelationships) {
             $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document));
         }
+
         $document = $this->decode($collection, $document);
 
         $this->trigger(self::EVENT_DOCUMENT_CREATE, $document);
+
         return $document;
     }
 
diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index be350d296..bd7704f8e 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -126,6 +126,8 @@ public function setPreserveDates(bool $preserve): static
     {
         $this->delegate('setPreserveDates', [$preserve]);
 
+        $this->preserveDates = $preserve;
+
         return $this;
     }
 
@@ -133,6 +135,8 @@ public function enableValidation(): static
     {
         $this->delegate('enableValidation');
 
+        $this->validate = true;
+
         return $this;
     }
 
@@ -140,6 +144,8 @@ public function disableValidation(): static
     {
         $this->delegate('disableValidation');
 
+        $this->validate = false;
+
         return $this;
     }
 

From d6c7adfe16571d88bdc43fe26bcbda2faf0f81d9 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 20:29:11 +1200
Subject: [PATCH 018/100] Fix timestamps

---
 src/Database/Adapter/MariaDB.php |  2 +-
 src/Database/Mirror.php          | 36 ++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 369cadfa1..e2d552457 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -841,7 +841,7 @@ public function createDocument(string $collection, Document $document): Document
                     $stmtPermissions->bindValue(':_tenant', $this->tenant);
                 }
             }
-            
+
             $stmt->execute();
 
             $document['$internalId'] = $this->getDocument($collection, $document->getId(), [Query::select(['$internalId'])])->getInternalId();
diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index bd7704f8e..a6ae8b84e 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -259,6 +259,14 @@ public function deleteIndex(string $collection, string $id): bool
 
     public function createDocument(string $collection, Document $document): Document
     {
+        $time = DateTime::now();
+
+        $createdAt = $document->getCreatedAt();
+        $updatedAt = $document->getUpdatedAt();
+
+        $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
+        $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
+
         $document = $this->source->createDocument($collection, $document);
 
         if (
@@ -300,6 +308,16 @@ public function createDocuments(
         array $documents,
         int $batchSize = self::INSERT_BATCH_SIZE
     ): array {
+        $time = DateTime::now();
+
+        foreach ($documents as $document) {
+            $createdAt = $document->getCreatedAt();
+            $updatedAt = $document->getUpdatedAt();
+
+            $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
+            $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
+        }
+
         $documents = $this->source->createDocuments($collection, $documents, $batchSize);
 
         if (
@@ -344,6 +362,14 @@ public function createDocuments(
 
     public function updateDocument(string $collection, string $id, Document $document): Document
     {
+        $time = DateTime::now();
+
+        $createdAt = $document->getCreatedAt();
+        $updatedAt = $document->getUpdatedAt();
+
+        $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
+        $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
+
         $document = $this->source->updateDocument($collection, $id, $document);
 
         if (
@@ -385,6 +411,16 @@ public function updateDocuments(
         array $documents,
         int $batchSize = self::INSERT_BATCH_SIZE
     ): array {
+        $time = DateTime::now();
+
+        foreach ($documents as $document) {
+            $createdAt = $document->getCreatedAt();
+            $updatedAt = $document->getUpdatedAt();
+
+            $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
+            $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
+        }
+
         $documents = $this->source->updateDocuments($collection, $documents, $batchSize);
 
         if (

From ba308bd5c0932997de8e17e8855c41ac8b8dc776 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 7 Jun 2024 20:31:57 +1200
Subject: [PATCH 019/100] Add document crud tests

---
 tests/e2e/Adapter/MirrorTest.php | 159 ++++++++++++++++++++++++++++---
 1 file changed, 147 insertions(+), 12 deletions(-)

diff --git a/tests/e2e/Adapter/MirrorTest.php b/tests/e2e/Adapter/MirrorTest.php
index 258241945..8a0bef837 100644
--- a/tests/e2e/Adapter/MirrorTest.php
+++ b/tests/e2e/Adapter/MirrorTest.php
@@ -8,10 +8,13 @@
 use Utopia\Cache\Cache;
 use Utopia\Database\Adapter\MariaDB;
 use Utopia\Database\Database;
+use Utopia\Database\Document;
 use Utopia\Database\Exception;
+use Utopia\Database\Exception\Authorization;
 use Utopia\Database\Exception\Conflict;
 use Utopia\Database\Exception\Duplicate;
 use Utopia\Database\Exception\Limit;
+use Utopia\Database\Exception\Structure;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
 use Utopia\Database\Mirror;
@@ -80,7 +83,7 @@ protected static function getAdapterName(): string
      * @throws Exception
      * @throws \RedisException
      */
-    public function testGetSource(): void
+    public function testGetMirrorSource(): void
     {
         $database = self::getDatabase();
         $source = $database->getSource();
@@ -92,7 +95,7 @@ public function testGetSource(): void
      * @throws Exception
      * @throws \RedisException
      */
-    public function testGetDestination(): void
+    public function testGetMirrorDestination(): void
     {
         $database = self::getDatabase();
         $destination = $database->getDestination();
@@ -106,15 +109,15 @@ public function testGetDestination(): void
      * @throws Exception
      * @throws \RedisException
      */
-    public function testCreateCollection(): void
+    public function testCreateMirroredCollection(): void
     {
         $database = self::getDatabase();
 
-        $database->createCollection('testCreateCollection');
+        $database->createCollection('testCreateMirroredCollection');
 
         // Assert collection exists in both databases
-        $this->assertFalse($database->getSource()->getCollection('testCreateCollection')->isEmpty());
-        $this->assertFalse($database->getDestination()->getCollection('testCreateCollection')->isEmpty());
+        $this->assertFalse($database->getSource()->getCollection('testCreateMirroredCollection')->isEmpty());
+        $this->assertFalse($database->getDestination()->getCollection('testCreateMirroredCollection')->isEmpty());
     }
 
     /**
@@ -124,18 +127,18 @@ public function testCreateCollection(): void
      * @throws Conflict
      * @throws Exception
      */
-    public function testUpdateCollection(): void
+    public function testUpdateMirroredCollection(): void
     {
         $database = self::getDatabase();
 
-        $database->createCollection('testUpdateCollection', permissions: [
+        $database->createCollection('testUpdateMirroredCollection', permissions: [
             Permission::read(Role::any()),
         ]);
 
-        $collection = $database->getCollection('testUpdateCollection');
+        $collection = $database->getCollection('testUpdateMirroredCollection');
 
         $database->updateCollection(
-            'testUpdateCollection',
+            'testUpdateMirroredCollection',
             [
                 Permission::read(Role::users()),
             ],
@@ -145,12 +148,144 @@ public function testUpdateCollection(): void
         // Asset both databases have updated the collection
         $this->assertEquals(
             [Permission::read(Role::users())],
-            $database->getSource()->getCollection('testUpdateCollection')->getPermissions()
+            $database->getSource()->getCollection('testUpdateMirroredCollection')->getPermissions()
         );
 
         $this->assertEquals(
             [Permission::read(Role::users())],
-            $database->getDestination()->getCollection('testUpdateCollection')->getPermissions()
+            $database->getDestination()->getCollection('testUpdateMirroredCollection')->getPermissions()
         );
     }
+
+    public function testDeleteMirroredCollection(): void
+    {
+        $database = self::getDatabase();
+
+        $database->createCollection('testDeleteMirroredCollection');
+
+        $database->deleteCollection('testDeleteMirroredCollection');
+
+        // Assert collection is deleted in both databases
+        $this->assertTrue($database->getSource()->getCollection('testDeleteMirroredCollection')->isEmpty());
+        $this->assertTrue($database->getDestination()->getCollection('testDeleteMirroredCollection')->isEmpty());
+    }
+
+    /**
+     * @throws Authorization
+     * @throws Duplicate
+     * @throws \RedisException
+     * @throws Limit
+     * @throws Structure
+     * @throws Exception
+     */
+    public function testCreateMirroredDocument(): void
+    {
+        $database = self::getDatabase();
+
+        $database->createCollection('testCreateMirroredDocument', attributes: [
+            new Document([
+                '$id' => 'name',
+                'type' => Database::VAR_STRING,
+                'required' => true,
+                'size' => Database::LENGTH_KEY,
+            ]),
+        ], permissions: [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+        ], documentSecurity: false);
+
+        $document = $database->createDocument('testCreateMirroredDocument', new Document([
+            'name' => 'Jake',
+            '$permissions' => []
+        ]));
+
+        // Assert document is created in both databases
+        $this->assertEquals(
+            $document,
+            $database->getSource()->getDocument('testCreateMirroredDocument', $document->getId())
+        );
+
+        $this->assertEquals(
+            $document,
+            $database->getDestination()->getDocument('testCreateMirroredDocument', $document->getId())
+        );
+    }
+
+    /**
+     * @throws Authorization
+     * @throws Duplicate
+     * @throws \RedisException
+     * @throws Conflict
+     * @throws Limit
+     * @throws Structure
+     * @throws Exception
+     */
+    public function testUpdateMirroredDocument(): void
+    {
+        $database = self::getDatabase();
+
+        $database->createCollection('testUpdateMirroredDocument', attributes: [
+            new Document([
+                '$id' => 'name',
+                'type' => Database::VAR_STRING,
+                'required' => true,
+                'size' => Database::LENGTH_KEY,
+            ]),
+        ], permissions: [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+            Permission::update(Role::any()),
+        ], documentSecurity: false);
+
+        $document = $database->createDocument('testUpdateMirroredDocument', new Document([
+            'name' => 'Jake',
+            '$permissions' => []
+        ]));
+
+        $document = $database->updateDocument(
+            'testUpdateMirroredDocument',
+            $document->getId(),
+            $document->setAttribute('name', 'John')
+        );
+
+        // Assert document is updated in both databases
+        $this->assertEquals(
+            $document,
+            $database->getSource()->getDocument('testUpdateMirroredDocument', $document->getId())
+        );
+
+        $this->assertEquals(
+            $document,
+            $database->getDestination()->getDocument('testUpdateMirroredDocument', $document->getId())
+        );
+    }
+
+    public function testDeleteMirroredDocument(): void
+    {
+        $database = self::getDatabase();
+
+        $database->createCollection('testDeleteMirroredDocument', attributes: [
+            new Document([
+                '$id' => 'name',
+                'type' => Database::VAR_STRING,
+                'required' => true,
+                'size' => Database::LENGTH_KEY,
+            ]),
+        ], permissions: [
+            Permission::create(Role::any()),
+            Permission::read(Role::any()),
+            Permission::delete(Role::any()),
+        ], documentSecurity: false);
+
+        $document = $database->createDocument('testDeleteMirroredDocument', new Document([
+            'name' => 'Jake',
+            '$permissions' => []
+        ]));
+
+        $database->deleteDocument('testDeleteMirroredDocument', $document->getId());
+
+        // Assert document is deleted in both databases
+        $this->assertTrue($database->getSource()->getDocument('testDeleteMirroredDocument', $document->getId())->isEmpty());
+        $this->assertTrue($database->getDestination()->getDocument('testDeleteMirroredDocument', $document->getId())->isEmpty());
+    }
 }

From e1c484d24397c0e413062bc9c0c905b5ef6fceaa Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 10 Jun 2024 18:30:18 +1200
Subject: [PATCH 020/100] Revert "Fix timestamps"

This reverts commit d6c7adfe16571d88bdc43fe26bcbda2faf0f81d9.
---
 src/Database/Adapter/MariaDB.php |  2 +-
 src/Database/Mirror.php          | 36 --------------------------------
 2 files changed, 1 insertion(+), 37 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index e2d552457..369cadfa1 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -841,7 +841,7 @@ public function createDocument(string $collection, Document $document): Document
                     $stmtPermissions->bindValue(':_tenant', $this->tenant);
                 }
             }
-
+            
             $stmt->execute();
 
             $document['$internalId'] = $this->getDocument($collection, $document->getId(), [Query::select(['$internalId'])])->getInternalId();
diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index a6ae8b84e..bd7704f8e 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -259,14 +259,6 @@ public function deleteIndex(string $collection, string $id): bool
 
     public function createDocument(string $collection, Document $document): Document
     {
-        $time = DateTime::now();
-
-        $createdAt = $document->getCreatedAt();
-        $updatedAt = $document->getUpdatedAt();
-
-        $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
-        $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
-
         $document = $this->source->createDocument($collection, $document);
 
         if (
@@ -308,16 +300,6 @@ public function createDocuments(
         array $documents,
         int $batchSize = self::INSERT_BATCH_SIZE
     ): array {
-        $time = DateTime::now();
-
-        foreach ($documents as $document) {
-            $createdAt = $document->getCreatedAt();
-            $updatedAt = $document->getUpdatedAt();
-
-            $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
-            $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
-        }
-
         $documents = $this->source->createDocuments($collection, $documents, $batchSize);
 
         if (
@@ -362,14 +344,6 @@ public function createDocuments(
 
     public function updateDocument(string $collection, string $id, Document $document): Document
     {
-        $time = DateTime::now();
-
-        $createdAt = $document->getCreatedAt();
-        $updatedAt = $document->getUpdatedAt();
-
-        $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
-        $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
-
         $document = $this->source->updateDocument($collection, $id, $document);
 
         if (
@@ -411,16 +385,6 @@ public function updateDocuments(
         array $documents,
         int $batchSize = self::INSERT_BATCH_SIZE
     ): array {
-        $time = DateTime::now();
-
-        foreach ($documents as $document) {
-            $createdAt = $document->getCreatedAt();
-            $updatedAt = $document->getUpdatedAt();
-
-            $document->setAttribute('$createdAt', empty($createdAt) || !$this->preserveDates ? $time : $createdAt);
-            $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
-        }
-
         $documents = $this->source->updateDocuments($collection, $documents, $batchSize);
 
         if (

From de90a3fa61b0dfeaa01269b2bc90ba6e21d338cc Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 10 Jun 2024 21:16:58 +1200
Subject: [PATCH 021/100] Add mirror to tests

---
 .github/workflows/tests.yml      | 1 +
 src/Database/Adapter/MariaDB.php | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index b3dabf856..7b9f672af 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -77,6 +77,7 @@ jobs:
             Postgres,
             SQLite,
             MongoDB,
+            Mirror,
             SharedTables/MariaDB,
             SharedTables/MySQL,
             SharedTables/Postgres,
diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 369cadfa1..e2d552457 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -841,7 +841,7 @@ public function createDocument(string $collection, Document $document): Document
                     $stmtPermissions->bindValue(':_tenant', $this->tenant);
                 }
             }
-            
+
             $stmt->execute();
 
             $document['$internalId'] = $this->getDocument($collection, $document->getId(), [Query::select(['$internalId'])])->getInternalId();

From 5da0ff69e867ab01bfe9c6cc5bb4a2c419691ae7 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 12 Jun 2024 13:45:05 +1200
Subject: [PATCH 022/100] Add for each filter function

---
 src/Database/Mirror.php | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index bd7704f8e..dccd276b2 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -591,6 +591,17 @@ public function createUpgrades(): void
         }
     }
 
+    /**
+     * @param callable(Filter): void $callback
+     * @return void
+     */
+    public function forEachFilter(callable $callback): void
+    {
+        foreach ($this->writeFilters as $filter) {
+            $callback($filter);
+        }
+    }
+
     /**
      * @throws Exception
      */

From 01468e56a88713c621949eff6f87769cbb086d92 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 12 Jun 2024 14:06:17 +1200
Subject: [PATCH 023/100] Expose filters directly

---
 src/Database/Mirror.php | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index dccd276b2..82376df29 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -63,6 +63,14 @@ public function getDestination(): ?Database
         return $this->destination;
     }
 
+    /**
+     * @return array<Filter>
+     */
+    public function getWriteFilters(): array
+    {
+        return $this->writeFilters;
+    }
+
     /**
      * @param callable(string, \Throwable): void $callback
      * @return void
@@ -591,17 +599,6 @@ public function createUpgrades(): void
         }
     }
 
-    /**
-     * @param callable(Filter): void $callback
-     * @return void
-     */
-    public function forEachFilter(callable $callback): void
-    {
-        foreach ($this->writeFilters as $filter) {
-            $callback($filter);
-        }
-    }
-
     /**
      * @throws Exception
      */

From f99148a158148186d18c78bdc6b0954ce58dfd1c Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 12 Jun 2024 14:52:43 +1200
Subject: [PATCH 024/100] Add default method stubs

---
 src/Database/Mirror.php           | 248 +++++++++++++++++++++++++++++-
 src/Database/Mirroring/Filter.php | 176 ++++++++++++++++++++-
 2 files changed, 411 insertions(+), 13 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 82376df29..38668922c 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -209,6 +209,15 @@ public function createCollection(string $id, array $attributes = [], array $inde
         }
 
         try {
+            foreach ($this->writeFilters as $filter) {
+                $result = $filter->onCreateCollection(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $id,
+                    collection: $result,
+                );
+            }
+
             $this->destination->createCollection(
                 $id,
                 $attributes,
@@ -232,37 +241,262 @@ public function createCollection(string $id, array $attributes = [], array $inde
 
     public function updateCollection(string $id, array $permissions, bool $documentSecurity): Document
     {
-        return $this->delegate('updateCollection', [$id, $permissions, $documentSecurity]);
+        $result = $this->source->updateCollection($id, $permissions, $documentSecurity);
+
+        if ($this->destination === null) {
+            return $result;
+        }
+
+        try {
+            foreach ($this->writeFilters as $filter) {
+                $result = $filter->onUpdateCollection(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $id,
+                    collection: $result,
+                );
+            }
+
+            $this->destination->updateCollection($id, $permissions, $documentSecurity);
+        } catch (\Throwable $err) {
+            $this->logError('updateCollection', $err);
+        }
+
+        return $result;
     }
 
     public function deleteCollection(string $id): bool
     {
-        return $this->delegate('deleteCollection', [$id]);
+        $result = $this->source->deleteCollection($id);
+
+        if ($this->destination === null) {
+            return $result;
+        }
+
+        try {
+            $this->destination->deleteCollection($id);
+
+            foreach ($this->writeFilters as $filter) {
+                $filter->onDeleteCollection(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $id,
+                );
+            }
+        } catch (\Throwable $err) {
+            $this->logError('deleteCollection', $err);
+        }
+
+        return $result;
     }
 
     public function createAttribute(string $collection, string $id, string $type, int $size, bool $required, $default = null, bool $signed = true, bool $array = false, string $format = null, array $formatOptions = [], array $filters = []): bool
     {
-        return $this->delegate('createAttribute', [$collection, $id, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters]);
+        $result = $this->source->createAttribute(
+            $collection,
+            $id,
+            $type,
+            $size,
+            $required,
+            $default,
+            $signed,
+            $array,
+            $format,
+            $formatOptions,
+            $filters
+        );
+
+        if ($this->destination === null) {
+            return $result;
+        }
+
+        try {
+            $document = new Document([
+                '$id' => $id,
+                'type' => $type,
+                'size' => $size,
+                'required' => $required,
+                'default' => $default,
+                'signed' => $signed,
+                'array' => $array,
+                'format' => $format,
+                'formatOptions' => $formatOptions,
+                'filters' => $filters,
+            ]);
+
+            foreach ($this->writeFilters as $filter) {
+                $document = $filter->onCreateAttribute(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    attributeId: $id,
+                    attribute: $document,
+                );
+            }
+
+           $result = $this->destination->createAttribute(
+                $collection,
+                $document->getId(),
+                $document->getAttribute('type'),
+                $document->getAttribute('size'),
+                $document->getAttribute('required'),
+                $document->getAttribute('default'),
+                $document->getAttribute('signed'),
+                $document->getAttribute('array'),
+                $document->getAttribute('format'),
+                $document->getAttribute('formatOptions'),
+                $document->getAttribute('filters'),
+            );
+        } catch (\Throwable $err) {
+            $this->logError('createAttribute', $err);
+        }
+
+        return $result;
     }
 
     public function updateAttribute(string $collection, string $id, string $type = null, int $size = null, bool $required = null, mixed $default = null, bool $signed = null, bool $array = null, string $format = null, ?array $formatOptions = null, ?array $filters = null): Document
     {
-        return $this->delegate('updateAttribute', [$collection, $id, $type, $size, $required, $default, $signed, $array, $format, $formatOptions, $filters]);
+        $document = $this->source->updateAttribute(
+            $collection,
+            $id,
+            $type,
+            $size,
+            $required,
+            $default,
+            $signed,
+            $array,
+            $format,
+            $formatOptions,
+            $filters
+        );
+
+        if ($this->destination === null) {
+            return $document;
+        }
+
+        try {
+            foreach ($this->writeFilters as $filter) {
+                $document = $filter->onUpdateAttribute(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    attributeId: $id,
+                    attribute: $document,
+                );
+            }
+
+            $this->destination->updateAttribute(
+                $collection,
+                $id,
+                $document->getAttribute('type'),
+                $document->getAttribute('size'),
+                $document->getAttribute('required'),
+                $document->getAttribute('default'),
+                $document->getAttribute('signed'),
+                $document->getAttribute('array'),
+                $document->getAttribute('format'),
+                $document->getAttribute('formatOptions'),
+                $document->getAttribute('filters')
+            );
+        } catch (\Throwable $err) {
+            $this->logError('updateAttribute', $err);
+        }
+
+        return $document;
     }
 
     public function deleteAttribute(string $collection, string $id): bool
     {
-        return $this->delegate('deleteAttribute', [$collection, $id]);
+        $result = $this->source->deleteAttribute($collection, $id);
+
+        if ($this->destination === null) {
+            return $result;
+        }
+
+        try {
+            foreach ($this->writeFilters as $filter) {
+                $filter->onDeleteAttribute(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    attributeId: $id,
+                );
+            }
+
+            $this->destination->deleteAttribute($collection, $id);
+        } catch (\Throwable $err) {
+            $this->logError('deleteAttribute', $err);
+        }
+
+        return $result;
     }
 
     public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool
     {
-        return $this->delegate('createIndex', [$collection, $id, $type, $attributes, $lengths, $orders]);
+        $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders);
+
+        if ($this->destination === null) {
+            return $result;
+        }
+
+        try {
+            $document = new Document([
+                '$id' => $id,
+                'type' => $type,
+                'attributes' => $attributes,
+                'lengths' => $lengths,
+                'orders' => $orders,
+            ]);
+
+            foreach ($this->writeFilters as $filter) {
+                $document = $filter->onCreateIndex(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    indexId: $id,
+                    index: $document,
+                );
+            }
+
+            $result = $this->destination->createIndex(
+                $collection,
+                $document->getId(),
+                $document->getAttribute('type'),
+                $document->getAttribute('attributes'),
+                $document->getAttribute('lengths'),
+                $document->getAttribute('orders')
+            );
+        } catch (\Throwable $err) {
+            $this->logError('createIndex', $err);
+        }
+
+        return $result;
     }
 
     public function deleteIndex(string $collection, string $id): bool
     {
-        return $this->delegate('deleteIndex', [$collection, $id]);
+        $result = $this->source->deleteIndex($collection, $id);
+
+        if ($this->destination === null) {
+            return $result;
+        }
+
+        try {
+            $this->destination->deleteIndex($collection, $id);
+
+            foreach ($this->writeFilters as $filter) {
+                $filter->onDeleteIndex(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    indexId: $id,
+                );
+            }
+        } catch (\Throwable $err) {
+            $this->logError('deleteIndex', $err);
+        }
+
+        return $result;
     }
 
     public function createDocument(string $collection, Document $document): Document
diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index f247b9220..2464c4d34 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -7,6 +7,164 @@
 
 abstract class Filter
 {
+    /**
+     * Called before collection is created in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param Document $collection
+     * @return Document
+     */
+    public function onCreateCollection(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        Document $collection,
+    ): Document {
+        return $collection;
+    }
+
+    /**
+     * Called before collection is updated in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param Document $collection
+     * @return Document
+     */
+    public function onUpdateCollection(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        Document $collection,
+    ): Document {
+        return $collection;
+    }
+
+    /**
+     * Called after collection is deleted in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @return void
+     */
+    public function onDeleteCollection(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+    ): void {
+        return;
+    }
+
+    /**
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $attributeId
+     * @param Document $attribute
+     * @return Document
+     */
+    public function onCreateAttribute(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        string $attributeId,
+        Document $attribute,
+    ): Document {
+        return $attribute;
+    }
+
+    /**
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $attributeId
+     * @param Document $attribute
+     * @return Document
+     */
+    public function onUpdateAttribute(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        string $attributeId,
+        Document $attribute,
+    ): Document {
+        return $attribute;
+    }
+
+    /**
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $attributeId
+     * @return void
+     */
+    public function onDeleteAttribute(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        string $attributeId,
+    ): void {
+        return;
+    }
+
+    // Indexes
+
+    /**
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $indexId
+     * @param Document $index
+     * @return Document
+     */
+    public function onCreateIndex(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        string $indexId,
+        Document $index,
+    ): Document {
+        return $index;
+    }
+
+    /**
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $indexId
+     * @param Document $index
+     * @return Document
+     */
+    public function onUpdateIndex(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        string $indexId,
+        Document $index,
+    ): Document {
+        return $index;
+    }
+
+    /**
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $indexId
+     * @return void
+     */
+    public function onDeleteIndex(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        string $indexId,
+    ): void {
+        return;
+    }
+
     /**
      * Called before document is created in the destination database
      *
@@ -16,12 +174,14 @@ abstract class Filter
      * @param Document $document
      * @return Document
      */
-    abstract public function onCreateDocument(
+    public function onCreateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
         Document $document,
-    ): Document;
+    ): Document {
+        return $document;
+    }
 
 
     /**
@@ -33,12 +193,14 @@ abstract public function onCreateDocument(
      * @param Document $document
      * @return Document
      */
-    abstract public function onUpdateDocument(
+    public function onUpdateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
         Document $document,
-    ): Document;
+    ): Document {
+        return $document;
+    }
 
     /**
      * Called after document is deleted in the destination database
@@ -49,10 +211,12 @@ abstract public function onUpdateDocument(
      * @param string $documentId
      * @return void
      */
-    abstract public function onDeleteDocument(
+    public function onDeleteDocument(
         Database $source,
         Database $destination,
         string $collectionId,
         string $documentId,
-    ): void;
+    ): void {
+        return;
+    }
 }

From 238e4a072604e2e32f47c712b8efa9c973e2e8b9 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 12 Jun 2024 15:17:42 +1200
Subject: [PATCH 025/100] Allow hooking after even as well as before

---
 src/Database/Mirror.php           | 73 +++++++++++++++++++++++------
 src/Database/Mirroring/Filter.php | 78 ++++++++++++++++++++++++-------
 2 files changed, 120 insertions(+), 31 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 38668922c..2911a0231 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -210,7 +210,7 @@ public function createCollection(string $id, array $attributes = [], array $inde
 
         try {
             foreach ($this->writeFilters as $filter) {
-                $result = $filter->onCreateCollection(
+                $result = $filter->beforeCreateCollection(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $id,
@@ -249,7 +249,7 @@ public function updateCollection(string $id, array $permissions, bool $documentS
 
         try {
             foreach ($this->writeFilters as $filter) {
-                $result = $filter->onUpdateCollection(
+                $result = $filter->beforeUpdateCollection(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $id,
@@ -277,7 +277,7 @@ public function deleteCollection(string $id): bool
             $this->destination->deleteCollection($id);
 
             foreach ($this->writeFilters as $filter) {
-                $filter->onDeleteCollection(
+                $filter->beforeDeleteCollection(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $id,
@@ -325,7 +325,7 @@ public function createAttribute(string $collection, string $id, string $type, in
             ]);
 
             foreach ($this->writeFilters as $filter) {
-                $document = $filter->onCreateAttribute(
+                $document = $filter->beforeCreateAttribute(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
@@ -334,7 +334,7 @@ public function createAttribute(string $collection, string $id, string $type, in
                 );
             }
 
-           $result = $this->destination->createAttribute(
+            $result = $this->destination->createAttribute(
                 $collection,
                 $document->getId(),
                 $document->getAttribute('type'),
@@ -376,7 +376,7 @@ public function updateAttribute(string $collection, string $id, string $type = n
 
         try {
             foreach ($this->writeFilters as $filter) {
-                $document = $filter->onUpdateAttribute(
+                $document = $filter->beforeUpdateAttribute(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
@@ -415,7 +415,7 @@ public function deleteAttribute(string $collection, string $id): bool
 
         try {
             foreach ($this->writeFilters as $filter) {
-                $filter->onDeleteAttribute(
+                $filter->beforeDeleteAttribute(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
@@ -449,7 +449,7 @@ public function createIndex(string $collection, string $id, string $type, array
             ]);
 
             foreach ($this->writeFilters as $filter) {
-                $document = $filter->onCreateIndex(
+                $document = $filter->beforeCreateIndex(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
@@ -485,7 +485,7 @@ public function deleteIndex(string $collection, string $id): bool
             $this->destination->deleteIndex($collection, $id);
 
             foreach ($this->writeFilters as $filter) {
-                $filter->onDeleteIndex(
+                $filter->beforeDeleteIndex(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
@@ -519,7 +519,7 @@ public function createDocument(string $collection, Document $document): Document
             $clone = clone $document;
 
             foreach ($this->writeFilters as $filter) {
-                $clone = $filter->onCreateDocument(
+                $clone = $filter->beforeCreateDocument(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
@@ -528,8 +528,17 @@ public function createDocument(string $collection, Document $document): Document
             }
 
             $this->destination->setPreserveDates(true);
-            $this->destination->createDocument($collection, $clone);
+            $document = $this->destination->createDocument($collection, $clone);
             $this->destination->setPreserveDates(false);
+
+            foreach ($this->writeFilters as $filter) {
+                $filter->afterCreateDocument(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    document: $clone,
+                );
+            }
         } catch (\Throwable $err) {
             $this->logError('createDocument', $err);
         }
@@ -563,7 +572,7 @@ public function createDocuments(
                 $clone = clone $document;
 
                 foreach ($this->writeFilters as $filter) {
-                    $clone = $filter->onCreateDocument(
+                    $clone = $filter->beforeCreateDocument(
                         source: $this->source,
                         destination: $this->destination,
                         collectionId: $collection,
@@ -577,6 +586,18 @@ public function createDocuments(
             $this->destination->setPreserveDates(true);
             $this->destination->createDocuments($collection, $clones, $batchSize);
             $this->destination->setPreserveDates(false);
+
+            foreach ($clones as $clone) {
+                foreach ($this->writeFilters as $filter) {
+                    $filter->afterCreateDocument(
+                        source: $this->source,
+                        destination: $this->destination,
+                        collectionId: $collection,
+                        document: $clone,
+                    );
+                }
+            }
+
         } catch (\Throwable $err) {
             $this->logError('createDocuments', $err);
         }
@@ -604,7 +625,7 @@ public function updateDocument(string $collection, string $id, Document $documen
             $clone = clone $document;
 
             foreach ($this->writeFilters as $filter) {
-                $clone = $filter->onUpdateDocument(
+                $clone = $filter->beforeUpdateDocument(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
@@ -615,6 +636,15 @@ public function updateDocument(string $collection, string $id, Document $documen
             $this->destination->setPreserveDates(true);
             $this->destination->updateDocument($collection, $id, $clone);
             $this->destination->setPreserveDates(false);
+
+            foreach ($this->writeFilters as $filter) {
+                $filter->afterUpdateDocument(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    document: $clone,
+                );
+            }
         } catch (\Throwable $err) {
             $this->logError('updateDocument', $err);
         }
@@ -648,7 +678,7 @@ public function updateDocuments(
                 $clone = clone $document;
 
                 foreach ($this->writeFilters as $filter) {
-                    $clone = $filter->onUpdateDocument(
+                    $clone = $filter->beforeUpdateDocument(
                         source: $this->source,
                         destination: $this->destination,
                         collectionId: $collection,
@@ -662,6 +692,17 @@ public function updateDocuments(
             $this->destination->setPreserveDates(true);
             $this->destination->updateDocuments($collection, $clones, $batchSize);
             $this->destination->setPreserveDates(false);
+
+            foreach ($clones as $clone) {
+                foreach ($this->writeFilters as $filter) {
+                    $filter->afterUpdateDocument(
+                        source: $this->source,
+                        destination: $this->destination,
+                        collectionId: $collection,
+                        document: $clone,
+                    );
+                }
+            }
         } catch (\Throwable $err) {
             $this->logError('updateDocuments', $err);
         }
@@ -686,10 +727,12 @@ public function deleteDocument(string $collection, string $id): bool
         }
 
         try {
+
+
             $this->destination->deleteDocument($collection, $id);
 
             foreach ($this->writeFilters as $filter) {
-                $filter->onDeleteDocument(
+                $filter->afterDeleteDocument(
                     source: $this->source,
                     destination: $this->destination,
                     collectionId: $collection,
diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index 2464c4d34..ec9c30379 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -16,7 +16,7 @@ abstract class Filter
      * @param Document $collection
      * @return Document
      */
-    public function onCreateCollection(
+    public function beforeCreateCollection(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -34,7 +34,7 @@ public function onCreateCollection(
      * @param Document $collection
      * @return Document
      */
-    public function onUpdateCollection(
+    public function beforeUpdateCollection(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -51,12 +51,11 @@ public function onUpdateCollection(
      * @param string $collectionId
      * @return void
      */
-    public function onDeleteCollection(
+    public function beforeDeleteCollection(
         Database $source,
         Database $destination,
         string $collectionId,
     ): void {
-        return;
     }
 
     /**
@@ -67,7 +66,7 @@ public function onDeleteCollection(
      * @param Document $attribute
      * @return Document
      */
-    public function onCreateAttribute(
+    public function beforeCreateAttribute(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -85,7 +84,7 @@ public function onCreateAttribute(
      * @param Document $attribute
      * @return Document
      */
-    public function onUpdateAttribute(
+    public function beforeUpdateAttribute(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -102,13 +101,12 @@ public function onUpdateAttribute(
      * @param string $attributeId
      * @return void
      */
-    public function onDeleteAttribute(
+    public function beforeDeleteAttribute(
         Database $source,
         Database $destination,
         string $collectionId,
         string $attributeId,
     ): void {
-        return;
     }
 
     // Indexes
@@ -121,7 +119,7 @@ public function onDeleteAttribute(
      * @param Document $index
      * @return Document
      */
-    public function onCreateIndex(
+    public function beforeCreateIndex(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -139,7 +137,7 @@ public function onCreateIndex(
      * @param Document $index
      * @return Document
      */
-    public function onUpdateIndex(
+    public function beforeUpdateIndex(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -156,13 +154,12 @@ public function onUpdateIndex(
      * @param string $indexId
      * @return void
      */
-    public function onDeleteIndex(
+    public function beforeDeleteIndex(
         Database $source,
         Database $destination,
         string $collectionId,
         string $indexId,
     ): void {
-        return;
     }
 
     /**
@@ -174,7 +171,7 @@ public function onDeleteIndex(
      * @param Document $document
      * @return Document
      */
-    public function onCreateDocument(
+    public function beforeCreateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -183,6 +180,22 @@ public function onCreateDocument(
         return $document;
     }
 
+    /**
+     * Called before document is created in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param Document $document
+     * @return void
+     */
+    public function afterCreateDocument(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        Document $document,
+    ): void {
+    }
 
     /**
      * Called before document is updated in the destination database
@@ -193,7 +206,7 @@ public function onCreateDocument(
      * @param Document $document
      * @return Document
      */
-    public function onUpdateDocument(
+    public function beforeUpdateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
@@ -202,6 +215,40 @@ public function onUpdateDocument(
         return $document;
     }
 
+    /**
+     * Called after document is updated in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param Document $document
+     * @return void
+     */
+    public function afterUpdateDocument(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        Document $document,
+    ): void {
+    }
+
+    /**
+     * Called before document is deleted in the destination database
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @param string $collectionId
+     * @param string $documentId
+     * @return void
+     */
+    public function beforeDeleteDocument(
+        Database $source,
+        Database $destination,
+        string $collectionId,
+        string $documentId,
+    ): void {
+    }
+
     /**
      * Called after document is deleted in the destination database
      *
@@ -211,12 +258,11 @@ public function onUpdateDocument(
      * @param string $documentId
      * @return void
      */
-    public function onDeleteDocument(
+    public function afterDeleteDocument(
         Database $source,
         Database $destination,
         string $collectionId,
         string $documentId,
     ): void {
-        return;
     }
 }

From a358f8d37b10e3f90eed70b7706ce0587dba886f Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 12 Jun 2024 15:55:56 +1200
Subject: [PATCH 026/100] Hook before delete

---
 src/Database/Mirror.php           | 9 ++++++++-
 src/Database/Mirroring/Filter.php | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 2911a0231..f4a3ee8ad 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -727,7 +727,14 @@ public function deleteDocument(string $collection, string $id): bool
         }
 
         try {
-
+            foreach ($this->writeFilters as $filter) {
+                $filter->beforeDeleteDocument(
+                    source: $this->source,
+                    destination: $this->destination,
+                    collectionId: $collection,
+                    documentId: $id,
+                );
+            }
 
             $this->destination->deleteDocument($collection, $id);
 
diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index ec9c30379..e239d5dca 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -181,7 +181,7 @@ public function beforeCreateDocument(
     }
 
     /**
-     * Called before document is created in the destination database
+     * Called after document is created in the destination database
      *
      * @param Database $source
      * @param Database $destination

From 60b9f1081b59e127dae35165dcf28217b4637678 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 12 Jun 2024 20:27:40 +1200
Subject: [PATCH 027/100] Add init and shutdown hooks to filters

---
 src/Database/Mirror.php           | 11 +++++++++++
 src/Database/Mirroring/Filter.php | 19 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index f4a3ee8ad..650d2e2a4 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -51,6 +51,17 @@ public function __construct(
         $this->source = $source;
         $this->destination = $destination;
         $this->writeFilters = $filters;
+
+        foreach ($this->writeFilters as $filter) {
+            $filter->init($this->source, $this->destination);
+        }
+    }
+
+    public function __destruct()
+    {
+        foreach ($this->writeFilters as $filter) {
+            $filter->shutdown($this->source, $this->destination);
+        }
     }
 
     public function getSource(): Database
diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index e239d5dca..205fd70ac 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -7,6 +7,25 @@
 
 abstract class Filter
 {
+    /**
+     * Called before any action is executed
+     *
+     * @param Database $source
+     * @param Database $destination
+     * @return void
+     */
+    public function init(
+        Database $source,
+        Database $destination,
+    ): void {
+    }
+
+    public function shutdown(
+        Database $source,
+        Database $destination,
+    ): void {
+    }
+
     /**
      * Called before collection is created in the destination database
      *

From 1ffa28554259e031e70d61d4da8f070093d4a74a Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 2 Jul 2024 13:44:29 +1200
Subject: [PATCH 028/100] Ensure destination is nullable

---
 src/Database/Mirroring/Filter.php       | 75 ++++++++++++++-----------
 src/Database/Validator/Query/Filter.php |  1 +
 2 files changed, 42 insertions(+), 34 deletions(-)

diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index 205fd70ac..f291c7042 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -8,21 +8,28 @@
 abstract class Filter
 {
     /**
-     * Called before any action is executed
+     * Called before any action is executed, when the filter is constructed.
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @return void
      */
     public function init(
         Database $source,
-        Database $destination,
+        ?Database $destination,
     ): void {
     }
 
+    /**
+     * Called after all actions are executed, when the filter is destructed.
+     *
+     * @param Database $source
+     * @param ?Database $destination
+     * @return void
+     */
     public function shutdown(
         Database $source,
-        Database $destination,
+        ?Database $destination,
     ): void {
     }
 
@@ -30,14 +37,14 @@ public function shutdown(
      * Called before collection is created in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param Document $collection
      * @return Document
      */
     public function beforeCreateCollection(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         Document $collection,
     ): Document {
@@ -48,14 +55,14 @@ public function beforeCreateCollection(
      * Called before collection is updated in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param Document $collection
      * @return Document
      */
     public function beforeUpdateCollection(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         Document $collection,
     ): Document {
@@ -66,20 +73,20 @@ public function beforeUpdateCollection(
      * Called after collection is deleted in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @return void
      */
     public function beforeDeleteCollection(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
     ): void {
     }
 
     /**
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $attributeId
      * @param Document $attribute
@@ -87,7 +94,7 @@ public function beforeDeleteCollection(
      */
     public function beforeCreateAttribute(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $attributeId,
         Document $attribute,
@@ -97,7 +104,7 @@ public function beforeCreateAttribute(
 
     /**
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $attributeId
      * @param Document $attribute
@@ -105,7 +112,7 @@ public function beforeCreateAttribute(
      */
     public function beforeUpdateAttribute(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $attributeId,
         Document $attribute,
@@ -115,14 +122,14 @@ public function beforeUpdateAttribute(
 
     /**
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $attributeId
      * @return void
      */
     public function beforeDeleteAttribute(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $attributeId,
     ): void {
@@ -132,7 +139,7 @@ public function beforeDeleteAttribute(
 
     /**
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $indexId
      * @param Document $index
@@ -140,7 +147,7 @@ public function beforeDeleteAttribute(
      */
     public function beforeCreateIndex(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $indexId,
         Document $index,
@@ -150,7 +157,7 @@ public function beforeCreateIndex(
 
     /**
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $indexId
      * @param Document $index
@@ -158,7 +165,7 @@ public function beforeCreateIndex(
      */
     public function beforeUpdateIndex(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $indexId,
         Document $index,
@@ -168,14 +175,14 @@ public function beforeUpdateIndex(
 
     /**
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $indexId
      * @return void
      */
     public function beforeDeleteIndex(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $indexId,
     ): void {
@@ -185,14 +192,14 @@ public function beforeDeleteIndex(
      * Called before document is created in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return Document
      */
     public function beforeCreateDocument(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         Document $document,
     ): Document {
@@ -203,14 +210,14 @@ public function beforeCreateDocument(
      * Called after document is created in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return void
      */
     public function afterCreateDocument(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         Document $document,
     ): void {
@@ -220,14 +227,14 @@ public function afterCreateDocument(
      * Called before document is updated in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return Document
      */
     public function beforeUpdateDocument(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         Document $document,
     ): Document {
@@ -238,14 +245,14 @@ public function beforeUpdateDocument(
      * Called after document is updated in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return void
      */
     public function afterUpdateDocument(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         Document $document,
     ): void {
@@ -255,14 +262,14 @@ public function afterUpdateDocument(
      * Called before document is deleted in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $documentId
      * @return void
      */
     public function beforeDeleteDocument(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $documentId,
     ): void {
@@ -272,14 +279,14 @@ public function beforeDeleteDocument(
      * Called after document is deleted in the destination database
      *
      * @param Database $source
-     * @param Database $destination
+     * @param ?Database $destination
      * @param string $collectionId
      * @param string $documentId
      * @return void
      */
     public function afterDeleteDocument(
         Database $source,
-        Database $destination,
+        ?Database $destination,
         string $collectionId,
         string $documentId,
     ): void {
diff --git a/src/Database/Validator/Query/Filter.php b/src/Database/Validator/Query/Filter.php
index 635fa4732..f5027e6c5 100644
--- a/src/Database/Validator/Query/Filter.php
+++ b/src/Database/Validator/Query/Filter.php
@@ -67,6 +67,7 @@ protected function isValidAttribute(string $attribute): bool
     /**
      * @param string $attribute
      * @param array<mixed> $values
+     * @param string $method
      * @return bool
      */
     protected function isValidAttributeAndValues(string $attribute, array $values, string $method): bool

From 28e41327874f849427a2b9c2f02d9d1a70d6b0cb Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 2 Jul 2024 13:48:23 +1200
Subject: [PATCH 029/100] Nullable dest only for init and shutdown

---
 src/Database/Mirroring/Filter.php | 60 +++++++++++++++----------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index f291c7042..ce381dfec 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -37,14 +37,14 @@ public function shutdown(
      * Called before collection is created in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param Document $collection
      * @return Document
      */
     public function beforeCreateCollection(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         Document $collection,
     ): Document {
@@ -55,14 +55,14 @@ public function beforeCreateCollection(
      * Called before collection is updated in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param Document $collection
      * @return Document
      */
     public function beforeUpdateCollection(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         Document $collection,
     ): Document {
@@ -73,20 +73,20 @@ public function beforeUpdateCollection(
      * Called after collection is deleted in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @return void
      */
     public function beforeDeleteCollection(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
     ): void {
     }
 
     /**
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $attributeId
      * @param Document $attribute
@@ -94,7 +94,7 @@ public function beforeDeleteCollection(
      */
     public function beforeCreateAttribute(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $attributeId,
         Document $attribute,
@@ -104,7 +104,7 @@ public function beforeCreateAttribute(
 
     /**
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $attributeId
      * @param Document $attribute
@@ -112,7 +112,7 @@ public function beforeCreateAttribute(
      */
     public function beforeUpdateAttribute(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $attributeId,
         Document $attribute,
@@ -122,14 +122,14 @@ public function beforeUpdateAttribute(
 
     /**
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $attributeId
      * @return void
      */
     public function beforeDeleteAttribute(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $attributeId,
     ): void {
@@ -139,7 +139,7 @@ public function beforeDeleteAttribute(
 
     /**
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $indexId
      * @param Document $index
@@ -147,7 +147,7 @@ public function beforeDeleteAttribute(
      */
     public function beforeCreateIndex(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $indexId,
         Document $index,
@@ -157,7 +157,7 @@ public function beforeCreateIndex(
 
     /**
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $indexId
      * @param Document $index
@@ -165,7 +165,7 @@ public function beforeCreateIndex(
      */
     public function beforeUpdateIndex(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $indexId,
         Document $index,
@@ -175,14 +175,14 @@ public function beforeUpdateIndex(
 
     /**
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $indexId
      * @return void
      */
     public function beforeDeleteIndex(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $indexId,
     ): void {
@@ -192,14 +192,14 @@ public function beforeDeleteIndex(
      * Called before document is created in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return Document
      */
     public function beforeCreateDocument(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         Document $document,
     ): Document {
@@ -210,14 +210,14 @@ public function beforeCreateDocument(
      * Called after document is created in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return void
      */
     public function afterCreateDocument(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         Document $document,
     ): void {
@@ -227,14 +227,14 @@ public function afterCreateDocument(
      * Called before document is updated in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return Document
      */
     public function beforeUpdateDocument(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         Document $document,
     ): Document {
@@ -245,14 +245,14 @@ public function beforeUpdateDocument(
      * Called after document is updated in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param Document $document
      * @return void
      */
     public function afterUpdateDocument(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         Document $document,
     ): void {
@@ -262,14 +262,14 @@ public function afterUpdateDocument(
      * Called before document is deleted in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $documentId
      * @return void
      */
     public function beforeDeleteDocument(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $documentId,
     ): void {
@@ -279,14 +279,14 @@ public function beforeDeleteDocument(
      * Called after document is deleted in the destination database
      *
      * @param Database $source
-     * @param ?Database $destination
+     * @param Database $destination
      * @param string $collectionId
      * @param string $documentId
      * @return void
      */
     public function afterDeleteDocument(
         Database $source,
-        ?Database $destination,
+        Database $destination,
         string $collectionId,
         string $documentId,
     ): void {

From 9801cf9eaa96a3c1005072b0874f612bd4926af3 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 10 Jul 2024 15:51:34 +1200
Subject: [PATCH 030/100] Don't init filters on construction to allow for
 pre-config

---
 src/Database/Mirror.php | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 650d2e2a4..f4a3ee8ad 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -51,17 +51,6 @@ public function __construct(
         $this->source = $source;
         $this->destination = $destination;
         $this->writeFilters = $filters;
-
-        foreach ($this->writeFilters as $filter) {
-            $filter->init($this->source, $this->destination);
-        }
-    }
-
-    public function __destruct()
-    {
-        foreach ($this->writeFilters as $filter) {
-            $filter->shutdown($this->source, $this->destination);
-        }
     }
 
     public function getSource(): Database

From ec87dfbe074acdf83c246df8b55e9a83b39aaf3e Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 11 Jul 2024 17:38:31 +1200
Subject: [PATCH 031/100] Try read instead of check duplicate upgrades
 collection

---
 src/Database/Mirror.php | 96 +++++++++++++++++++++--------------------
 1 file changed, 49 insertions(+), 47 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index f4a3ee8ad..ec0375e9c 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -834,53 +834,55 @@ public function decreaseDocumentAttribute(string $collection, string $id, string
      */
     public function createUpgrades(): void
     {
-        try {
-            $this->source->createCollection(
-                id: 'upgrades',
-                attributes: [
-                    new Document([
-                        '$id' => ID::custom('collectionId'),
-                        'type' => Database::VAR_STRING,
-                        'size' => Database::LENGTH_KEY,
-                        'required' => true,
-                        'signed' => true,
-                        'array' => false,
-                        'filters' => [],
-                        'default' => null,
-                        'format' => ''
-                    ]),
-                    new Document([
-                        '$id' => ID::custom('status'),
-                        'type' => Database::VAR_STRING,
-                        'size' => Database::LENGTH_KEY,
-                        'required' => false,
-                        'signed' => true,
-                        'array' => false,
-                        'filters' => [],
-                        'default' => null,
-                        'format' => ''
-                    ]),
-                ],
-                indexes: [
-                    new Document([
-                        '$id' => ID::custom('_unique_collection'),
-                        'type' => Database::INDEX_UNIQUE,
-                        'attributes' => ['collectionId'],
-                        'lengths' => [Database::LENGTH_KEY],
-                        'orders' => [],
-                    ]),
-                    new Document([
-                        '$id' => ID::custom('_status_index'),
-                        'type' => Database::INDEX_KEY,
-                        'attributes' => ['status'],
-                        'lengths' => [Database::LENGTH_KEY],
-                        'orders' => [Database::ORDER_ASC],
-                    ]),
-                ],
-            );
-        } catch (DuplicateException) {
-            // Ignore
-        }
+        $collection = $this->source->getCollection('upgrades');
+
+        if (!$collection->isEmpty()) {
+            return;
+        }
+
+        $this->source->createCollection(
+            id: 'upgrades',
+            attributes: [
+                new Document([
+                    '$id' => ID::custom('collectionId'),
+                    'type' => Database::VAR_STRING,
+                    'size' => Database::LENGTH_KEY,
+                    'required' => true,
+                    'signed' => true,
+                    'array' => false,
+                    'filters' => [],
+                    'default' => null,
+                    'format' => ''
+                ]),
+                new Document([
+                    '$id' => ID::custom('status'),
+                    'type' => Database::VAR_STRING,
+                    'size' => Database::LENGTH_KEY,
+                    'required' => false,
+                    'signed' => true,
+                    'array' => false,
+                    'filters' => [],
+                    'default' => null,
+                    'format' => ''
+                ]),
+            ],
+            indexes: [
+                new Document([
+                    '$id' => ID::custom('_unique_collection'),
+                    'type' => Database::INDEX_UNIQUE,
+                    'attributes' => ['collectionId'],
+                    'lengths' => [Database::LENGTH_KEY],
+                    'orders' => [],
+                ]),
+                new Document([
+                    '$id' => ID::custom('_status_index'),
+                    'type' => Database::INDEX_KEY,
+                    'attributes' => ['status'],
+                    'lengths' => [Database::LENGTH_KEY],
+                    'orders' => [Database::ORDER_ASC],
+                ]),
+            ],
+        );
     }
 
     /**

From 6c86f496633e34b4f15ba479d7854013ff6131c4 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 12 Jul 2024 00:28:23 +1200
Subject: [PATCH 032/100] Fetch internal ID for documents not created with it

---
 src/Database/Adapter/MariaDB.php | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index e2d552457..ea245d02f 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -896,6 +896,8 @@ public function createDocuments(string $collection, array $documents, int $batch
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, max(1, $batchSize));
 
+            $internalIds = [];
+
             foreach ($batches as $batch) {
                 $bindIndex = 0;
                 $batchKeys = [];
@@ -908,7 +910,9 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $attributes['_createdAt'] = $document->getCreatedAt();
                     $attributes['_updatedAt'] = $document->getUpdatedAt();
                     $attributes['_permissions'] = \json_encode($document->getPermissions());
+
                     if(!empty($document->getInternalId())) {
+                        $internalIds[$document->getId()] = true;
                         $attributes['_id'] = $document->getInternalId();
                     }
 
@@ -1007,6 +1011,16 @@ public function createDocuments(string $collection, array $documents, int $batch
             throw $e;
         }
 
+        foreach ($documents as $document) {
+            if(!isset($internalIds[$document->getId()])) {
+                $document['$internalId'] = $this->getDocument(
+                    $collection,
+                    $document->getId(),
+                    [Query::select(['$internalId'])]
+                )->getInternalId();
+            }
+        }
+
         return $documents;
     }
 

From 89a0079fb68087f014098583ab56331b49b27961 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 22 Jul 2024 23:13:49 +1200
Subject: [PATCH 033/100] Ignore relationships that are virtual attributes

---
 src/Database/Adapter/MariaDB.php | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index ea245d02f..cc6d337a7 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -91,6 +91,23 @@ public function createCollection(string $name, array $attributes = [], array $in
                 $attribute->getAttribute('array', false)
             );
 
+            // Ignore relationships with virtual attributes
+            if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) {
+                $options = $attribute->getAttribute('options', []);
+                $relationType = $options['relationType'] ?? null;
+                $twoWay = $options['twoWay'] ?? false;
+                $side = $options['side'] ?? null;
+
+                if (
+                    $relationType === Database::RELATION_MANY_TO_MANY
+                    || ($relationType === Database::RELATION_ONE_TO_ONE && !$twoWay && $side === Database::RELATION_SIDE_CHILD)
+                    || ($relationType === Database::RELATION_ONE_TO_MANY && $side === Database::RELATION_SIDE_PARENT)
+                    || ($relationType === Database::RELATION_MANY_TO_ONE && $side === Database::RELATION_SIDE_CHILD)
+                ) {
+                    continue;
+                }
+            }
+
             $attributeStrings[$key] = "`{$attrId}` {$attrType}, ";
         }
 
@@ -894,7 +911,7 @@ public function createDocuments(string $collection, array $documents, int $batch
             $this->getPDO()->beginTransaction();
 
             $name = $this->filter($collection);
-            $batches = \array_chunk($documents, max(1, $batchSize));
+            $batches = \array_chunk($documents, \max(1, $batchSize));
 
             $internalIds = [];
 

From 68c988186f3cb6403ad5cf6cb6711523dc8e9e71 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 23 Jul 2024 17:25:50 +1200
Subject: [PATCH 034/100] Nullable filter params for noop hooks

---
 src/Database/Mirroring/Filter.php | 82 ++++++++++++++++---------------
 1 file changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index ce381dfec..8bbc5779d 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -39,15 +39,15 @@ public function shutdown(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param Document $collection
-     * @return Document
+     * @param ?Document $collection
+     * @return ?Document
      */
     public function beforeCreateCollection(
         Database $source,
         Database $destination,
         string $collectionId,
-        Document $collection,
-    ): Document {
+        ?Document $collection = null,
+    ): ?Document {
         return $collection;
     }
 
@@ -57,15 +57,15 @@ public function beforeCreateCollection(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param Document $collection
-     * @return Document
+     * @param ?Document $collection
+     * @return ?Document
      */
     public function beforeUpdateCollection(
         Database $source,
         Database $destination,
         string $collectionId,
-        Document $collection,
-    ): Document {
+        ?Document $collection = null,
+    ): ?Document {
         return $collection;
     }
 
@@ -89,16 +89,16 @@ public function beforeDeleteCollection(
      * @param Database $destination
      * @param string $collectionId
      * @param string $attributeId
-     * @param Document $attribute
-     * @return Document
+     * @param ?Document $attribute
+     * @return ?Document
      */
     public function beforeCreateAttribute(
         Database $source,
         Database $destination,
         string $collectionId,
         string $attributeId,
-        Document $attribute,
-    ): Document {
+        ?Document $attribute = null,
+    ): ?Document {
         return $attribute;
     }
 
@@ -107,16 +107,16 @@ public function beforeCreateAttribute(
      * @param Database $destination
      * @param string $collectionId
      * @param string $attributeId
-     * @param Document $attribute
-     * @return Document
+     * @param ?Document $attribute
+     * @return ?Document
      */
     public function beforeUpdateAttribute(
         Database $source,
         Database $destination,
         string $collectionId,
         string $attributeId,
-        Document $attribute,
-    ): Document {
+        ?Document $attribute = null,
+    ): ?Document {
         return $attribute;
     }
 
@@ -142,16 +142,16 @@ public function beforeDeleteAttribute(
      * @param Database $destination
      * @param string $collectionId
      * @param string $indexId
-     * @param Document $index
-     * @return Document
+     * @param ?Document $index
+     * @return ?Document
      */
     public function beforeCreateIndex(
         Database $source,
         Database $destination,
         string $collectionId,
         string $indexId,
-        Document $index,
-    ): Document {
+        ?Document $index = null,
+    ): ?Document {
         return $index;
     }
 
@@ -160,16 +160,16 @@ public function beforeCreateIndex(
      * @param Database $destination
      * @param string $collectionId
      * @param string $indexId
-     * @param Document $index
-     * @return Document
+     * @param ?Document $index
+     * @return ?Document
      */
     public function beforeUpdateIndex(
         Database $source,
         Database $destination,
         string $collectionId,
         string $indexId,
-        Document $index,
-    ): Document {
+        ?Document $index = null,
+    ): ?Document {
         return $index;
     }
 
@@ -194,15 +194,15 @@ public function beforeDeleteIndex(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param Document $document
-     * @return Document
+     * @param ?Document $document
+     * @return ?Document
      */
     public function beforeCreateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        Document $document,
-    ): Document {
+        ?Document $document = null,
+    ): ?Document {
         return $document;
     }
 
@@ -212,15 +212,16 @@ public function beforeCreateDocument(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param Document $document
-     * @return void
+     * @param ?Document $document
+     * @return ?Document
      */
     public function afterCreateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        Document $document,
-    ): void {
+        ?Document $document = null,
+    ): ?Document {
+        return $document;
     }
 
     /**
@@ -229,15 +230,15 @@ public function afterCreateDocument(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param Document $document
-     * @return Document
+     * @param ?Document $document
+     * @return ?Document
      */
     public function beforeUpdateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        Document $document,
-    ): Document {
+        ?Document $document = null,
+    ): ?Document {
         return $document;
     }
 
@@ -247,15 +248,16 @@ public function beforeUpdateDocument(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param Document $document
-     * @return void
+     * @param ?Document $document
+     * @return ?Document
      */
     public function afterUpdateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        Document $document,
-    ): void {
+        ?Document $document = null,
+    ): ?Document {
+        return $document;
     }
 
     /**

From 6b0defbcb442507a5928afb7c40035f245e04d5c Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 23 Jul 2024 17:38:47 +1200
Subject: [PATCH 035/100] Foce require document for document hooks

---
 src/Database/Mirroring/Filter.php | 32 +++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/Database/Mirroring/Filter.php b/src/Database/Mirroring/Filter.php
index 8bbc5779d..159c6884c 100644
--- a/src/Database/Mirroring/Filter.php
+++ b/src/Database/Mirroring/Filter.php
@@ -194,15 +194,15 @@ public function beforeDeleteIndex(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param ?Document $document
-     * @return ?Document
+     * @param Document $document
+     * @return Document
      */
     public function beforeCreateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        ?Document $document = null,
-    ): ?Document {
+        Document $document,
+    ): Document {
         return $document;
     }
 
@@ -212,15 +212,15 @@ public function beforeCreateDocument(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param ?Document $document
-     * @return ?Document
+     * @param Document $document
+     * @return Document
      */
     public function afterCreateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        ?Document $document = null,
-    ): ?Document {
+        Document $document,
+    ): Document {
         return $document;
     }
 
@@ -230,15 +230,15 @@ public function afterCreateDocument(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param ?Document $document
-     * @return ?Document
+     * @param Document $document
+     * @return Document
      */
     public function beforeUpdateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        ?Document $document = null,
-    ): ?Document {
+        Document $document,
+    ): Document {
         return $document;
     }
 
@@ -248,15 +248,15 @@ public function beforeUpdateDocument(
      * @param Database $source
      * @param Database $destination
      * @param string $collectionId
-     * @param ?Document $document
-     * @return ?Document
+     * @param Document $document
+     * @return Document
      */
     public function afterUpdateDocument(
         Database $source,
         Database $destination,
         string $collectionId,
-        ?Document $document = null,
-    ): ?Document {
+        Document $document,
+    ): Document {
         return $document;
     }
 

From 397aaa6ed4e7ee87042c51b2a1d900de13100679 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 15:46:06 +1200
Subject: [PATCH 036/100] Allow null tenant to allow global tables with single
 metadata doc

---
 src/Database/Adapter/MariaDB.php  | 22 +++++++++++-----------
 src/Database/Adapter/Postgres.php | 22 +++++++++++-----------
 src/Database/Adapter/SQL.php      |  4 ++--
 src/Database/Adapter/SQLite.php   | 10 +++++-----
 src/Database/Database.php         | 26 +++++++++++++++-----------
 5 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 28dad079a..3506221b7 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -1046,7 +1046,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant';
+                $sql .= ' AND _tenant IN (:_tenant, NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1123,7 +1123,7 @@ public function updateDocument(string $collection, Document $document): Document
                 ";
 
                 if ($this->sharedTables) {
-                    $sql .= ' AND _tenant = :_tenant';
+                    $sql .= ' AND _tenant IN (:_tenant, NULL)';
                 }
 
                 $removeQuery = $sql . $removeQuery;
@@ -1210,7 +1210,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant';
+                $sql .= ' AND _tenant IN (:_tenant, NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -1339,7 +1339,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= ' AND _tenant = :_tenant';
+                        $sql .= ' AND _tenant IN (:_tenant, NULL)';
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1382,7 +1382,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant = :_tenant';
+                                $tenantQuery = ' AND _tenant IN (:_tenant, NULL)';
                             }
 
                             $removeQuery .= "(
@@ -1563,7 +1563,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant';
+            $sql .= ' AND _tenant IN (:_tenant, NULL)';
         }
 
         $sql .= $sqlMax . $sqlMin;
@@ -1605,7 +1605,7 @@ public function deleteDocument(string $collection, string $id): bool
 		    ";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant';
+                $sql .= ' AND _tenant IN (:_tenant, NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
@@ -1624,7 +1624,7 @@ public function deleteDocument(string $collection, string $id): bool
 		    ";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant';
+                $sql .= ' AND _tenant IN (:_tenant, NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql);
@@ -1767,7 +1767,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant = :_tenant";
+            $where[] = "table_main._tenant IN (:_tenant, NULL)";
         }
 
         $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : '';
@@ -1893,7 +1893,7 @@ public function count(string $collection, array $queries = [], ?int $max = null)
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant = :_tenant";
+            $where[] = "table_main._tenant IN (:_tenant, NULL)";
         }
 
         $sqlWhere = !empty($where)
@@ -1963,7 +1963,7 @@ public function sum(string $collection, string $attribute, array $queries = [],
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant = :_tenant";
+            $where[] = "table_main._tenant IN (:_tenant, NULL)";
         }
 
         $sqlWhere = !empty($where)
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index 8686f4b2e..b6e98b069 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -977,7 +977,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant';
+            $sql .= ' AND _tenant IN (:_tenant, NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1057,7 +1057,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant';
+                $sql .= ' AND _tenant IN (:_tenant, NULL)';
             }
 
             $removeQuery = $sql . $removeQuery;
@@ -1129,7 +1129,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant';
+            $sql .= ' AND _tenant IN (:_tenant, NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -1253,7 +1253,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= ' AND _tenant = :_tenant';
+                        $sql .= ' AND _tenant IN (:_tenant, NULL)';
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1296,7 +1296,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant = :_tenant';
+                                $tenantQuery = ' AND _tenant IN (:_tenant, NULL)';
                             }
 
                             $removeQuery .= "(
@@ -1467,7 +1467,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant';
+            $sql .= ' AND _tenant IN (:_tenant, NULL)';
         }
 
         $sql .= $sqlMax . $sqlMin;
@@ -1507,7 +1507,7 @@ public function deleteDocument(string $collection, string $id): bool
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant';
+            $sql .= ' AND _tenant IN (:_tenant, NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
@@ -1524,7 +1524,7 @@ public function deleteDocument(string $collection, string $id): bool
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant';
+            $sql .= ' AND _tenant IN (:_tenant, NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql);
@@ -1661,7 +1661,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant = :_tenant";
+            $where[] = "table_main._tenant IN (:_tenant, NULL)";
         }
 
         if (Authorization::$status) {
@@ -1787,7 +1787,7 @@ public function count(string $collection, array $queries = [], ?int $max = null)
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant = :_tenant";
+            $where[] = "table_main._tenant IN (:_tenant, NULL)";
         }
 
         if (Authorization::$status) {
@@ -1850,7 +1850,7 @@ public function sum(string $collection, string $attribute, array $queries = [],
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant = :_tenant";
+            $where[] = "table_main._tenant IN (:_tenant, NULL)";
         }
 
         if (Authorization::$status) {
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index 8662cc94f..fdb7d8bea 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -116,7 +116,7 @@ public function getDocument(string $collection, string $id, array $queries = [])
 		";
 
         if ($this->sharedTables) {
-            $sql .= "AND _tenant = :_tenant";
+            $sql .= "AND _tenant IN (:_tenant, NULL)";
         }
 
         $stmt = $this->getPDO()->prepare($sql);
@@ -862,7 +862,7 @@ protected function getSQLPermissionsCondition(string $collection, array $roles):
 
         $tenantQuery = '';
         if ($this->sharedTables) {
-            $tenantQuery = 'AND _tenant = :_tenant';
+            $tenantQuery = 'AND _tenant IN (:_tenant, NULL)';
         }
 
         return "table_main._uid IN (
diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index ccbac7905..115bd879c 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -601,7 +601,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= " AND _tenant = :_tenant";
+            $sql .= " AND _tenant IN (:_tenant, NULL)";
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -684,7 +684,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= " AND _tenant = :_tenant";
+                $sql .= " AND _tenant IN (:_tenant, NULL)";
             }
 
             $removeQuery = $sql . $removeQuery;
@@ -756,7 +756,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= " AND _tenant = :_tenant";
+            $sql .= " AND _tenant IN (:_tenant, NULL)";
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -879,7 +879,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= " AND _tenant = :_tenant";
+                        $sql .= " AND _tenant IN (:_tenant, NULL)";
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -926,7 +926,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant = :_tenant';
+                                $tenantQuery = ' AND _tenant IN (:_tenant, NULL)';
                             }
 
                             $removeQuery .= "(
diff --git a/src/Database/Database.php b/src/Database/Database.php
index a3912d411..3f1e40da2 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -970,10 +970,6 @@ public function delete(?string $database = null): bool
      */
     public function createCollection(string $id, array $attributes = [], array $indexes = [], array $permissions = null, bool $documentSecurity = true): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $permissions ??= [
             Permission::create(Role::any()),
         ];
@@ -1111,10 +1107,13 @@ public function getCollection(string $id): Document
 
         $collection = $this->silent(fn () => $this->getDocument(self::METADATA, $id));
 
+        $tenant = $collection->getAttribute('$tenant');
+
         if (
             $id !== self::METADATA
             && $this->adapter->getSharedTables()
-            && $collection->getAttribute('$tenant') != $this->adapter->getTenant()
+            && $tenant !== null
+            && $tenant != $this->adapter->getTenant()
         ) {
             return new Document();
         }
@@ -1144,11 +1143,12 @@ public function listCollections(int $limit = 25, int $offset = 0): array
             Query::offset($offset)
         ]));
 
-        if ($this->adapter->getSharedTables()) {
-            $result = \array_filter($result, function ($collection) {
-                return $collection->getAttribute('$tenant') == $this->adapter->getTenant();
-            });
-        }
+        // TODO: Should this be required?
+        //if ($this->adapter->getSharedTables()) {
+        //    $result = \array_filter($result, function ($collection) {
+        //        return $collection->getAttribute('$tenant') == $this->adapter->getTenant();
+        //    });
+        //}
 
         $this->trigger(self::EVENT_COLLECTION_LIST, $result);
 
@@ -3087,7 +3087,11 @@ private function populateDocumentRelationships(Document $collection, Document $d
      */
     public function createDocument(string $collection, Document $document): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
+        if (
+            $this->adapter->getSharedTables()
+            && empty($this->adapter->getTenant())
+            && $collection !== self::METADATA
+        ) {
             throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
         }
 

From fdc9da7c082936745bbed291b8edeab4cb448e72 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 19:54:03 +1200
Subject: [PATCH 037/100] Don't require tenant for create database

---
 src/Database/Database.php  | 5 +----
 tests/e2e/Adapter/Base.php | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 3f1e40da2..0c2bb2119 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -871,11 +871,8 @@ public function ping(): bool
      */
     public function create(?string $database = null): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
+        $database ??= $this->adapter->getDatabase();
 
-        $database = $database ?? $this->adapter->getDatabase();
         $this->adapter->create($database);
 
         /**
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index c9f495cb3..662a3ac59 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -15011,7 +15011,7 @@ public function testEmptyOperatorValues(): void
      * @throws StructureException
      * @throws TimeoutException
      */
-    public function testIsolationModes(): void
+    public function testSharedTables(): void
     {
         /**
          * Default mode already tested, we'll test 'schema' and 'table' isolation here

From 8b6e99205a324fbffbe79ee4b79f3a54357f0195 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 20:22:54 +1200
Subject: [PATCH 038/100] Dont require tenant on get metadata collection

---
 src/Database/Database.php | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 0c2bb2119..1ffbd377b 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -1098,7 +1098,11 @@ public function updateCollection(string $id, array $permissions, bool $documentS
      */
     public function getCollection(string $id): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
+        if (
+            $id !== self::METADATA
+            && $this->adapter->getSharedTables()
+            && empty($this->adapter->getTenant())
+        ) {
             throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
         }
 

From bcbee88f4a3a57affd19a0eb41ba297098dd2e7d Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 20:30:53 +1200
Subject: [PATCH 039/100] Fix check order for getDocument

---
 src/Database/Database.php | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 1ffbd377b..fb2020d74 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -2652,10 +2652,6 @@ public function deleteIndex(string $collection, string $id): bool
      */
     public function getDocument(string $collection, string $id, array $queries = []): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if ($collection === self::METADATA && $id === self::METADATA) {
             return new Document(self::COLLECTION);
         }
@@ -2664,6 +2660,10 @@ public function getDocument(string $collection, string $id, array $queries = [])
             throw new DatabaseException('Collection not found');
         }
 
+        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
+            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
+        }
+
         if (empty($id)) {
             return new Document();
         }

From 0b2cda2cb5a21d3bcbf14ae2fa2dfda6c1aa328c Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 20:46:14 +1200
Subject: [PATCH 040/100] Remove exists tenant check

---
 src/Database/Database.php | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index fb2020d74..dec567e1e 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -901,11 +901,7 @@ public function create(?string $database = null): bool
      */
     public function exists(?string $database = null, ?string $collection = null): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
-        $database = $database ?? $this->adapter->getDatabase();
+        $database ??= $this->adapter->getDatabase();
 
         return $this->adapter->exists($database, $collection);
     }

From 15dc0ae7d73646d5fb6d57aa830ecdd9c8ef82bd Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 21:01:04 +1200
Subject: [PATCH 041/100] Remove redundant missing tenant checks

---
 src/Database/Database.php | 116 +-------------------------------------
 1 file changed, 2 insertions(+), 114 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index dec567e1e..dd17fb514 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -913,10 +913,6 @@ public function exists(?string $database = null, ?string $collection = null): bo
      */
     public function list(): array
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $databases = $this->adapter->list();
 
         $this->trigger(self::EVENT_DATABASE_LIST, $databases);
@@ -932,10 +928,6 @@ public function list(): array
      */
     public function delete(?string $database = null): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $database = $database ?? $this->adapter->getDatabase();
 
         $deleted = $this->adapter->delete($database);
@@ -1049,10 +1041,6 @@ public function createCollection(string $id, array $attributes = [], array $inde
      */
     public function updateCollection(string $id, array $permissions, bool $documentSecurity): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if ($this->validate) {
             $validator = new Permissions();
             if (!$validator->isValid($permissions)) {
@@ -1131,10 +1119,6 @@ public function getCollection(string $id): Document
      */
     public function listCollections(int $limit = 25, int $offset = 0): array
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $result = $this->silent(fn () => $this->find(self::METADATA, [
             Query::limit($limit),
             Query::offset($offset)
@@ -1161,10 +1145,6 @@ public function listCollections(int $limit = 25, int $offset = 0): array
      */
     public function getSizeOfCollection(string $collection): int
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         if ($collection->isEmpty()) {
@@ -1188,10 +1168,6 @@ public function getSizeOfCollection(string $collection): int
      */
     public function deleteCollection(string $id): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getDocument(self::METADATA, $id));
 
         if ($collection->isEmpty()) {
@@ -1252,10 +1228,6 @@ public function deleteCollection(string $id): bool
      */
     public function createAttribute(string $collection, string $id, string $type, int $size, bool $required, mixed $default = null, bool $signed = true, bool $array = false, string $format = null, array $formatOptions = [], array $filters = []): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         if ($collection->isEmpty()) {
@@ -1438,10 +1410,6 @@ protected function validateDefaultTypes(string $type, mixed $default): void
      */
     protected function updateIndexMeta(string $collection, string $id, callable $updateCallback): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         if ($collection->getId() === self::METADATA) {
@@ -1481,10 +1449,6 @@ protected function updateIndexMeta(string $collection, string $id, callable $upd
      */
     protected function updateAttributeMeta(string $collection, string $id, callable $updateCallback): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         if ($collection->getId() === self::METADATA) {
@@ -1741,10 +1705,6 @@ public function updateAttribute(string $collection, string $id, string $type = n
      */
     public function checkAttribute(Document $collection, Document $attribute): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = clone $collection;
 
         $collection->setAttribute('attributes', $attribute, Document::SET_TYPE_APPEND);
@@ -1778,10 +1738,6 @@ public function checkAttribute(Document $collection, Document $attribute): bool
      */
     public function deleteAttribute(string $collection, string $id): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
         $attributes = $collection->getAttribute('attributes', []);
         $indexes = $collection->getAttribute('indexes', []);
@@ -1852,10 +1808,6 @@ public function deleteAttribute(string $collection, string $id): bool
      */
     public function renameAttribute(string $collection, string $old, string $new): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
         $attributes = $collection->getAttribute('attributes', []);
         $indexes = $collection->getAttribute('indexes', []);
@@ -1930,10 +1882,6 @@ public function createRelationship(
         ?string $twoWayKey = null,
         string $onDelete = Database::RELATION_MUTATE_RESTRICT
     ): bool {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         if ($collection->isEmpty()) {
@@ -2122,10 +2070,6 @@ public function updateRelationship(
         ?bool $twoWay = null,
         ?string $onDelete = null
     ): bool {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if (
             \is_null($newKey)
             && \is_null($newTwoWayKey)
@@ -2304,10 +2248,6 @@ public function updateRelationship(
      */
     public function deleteRelationship(string $collection, string $id): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
         $attributes = $collection->getAttribute('attributes', []);
         $relationship = null;
@@ -2432,10 +2372,6 @@ public function deleteRelationship(string $collection, string $id): bool
      */
     public function renameIndex(string $collection, string $old, string $new): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         $indexes = $collection->getAttribute('indexes', []);
@@ -2495,10 +2431,6 @@ public function renameIndex(string $collection, string $old, string $new): bool
      */
     public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if (empty($attributes)) {
             throw new DatabaseException('Missing attributes');
         }
@@ -2606,10 +2538,6 @@ public function createIndex(string $collection, string $id, string $type, array
      */
     public function deleteIndex(string $collection, string $id): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         $indexes = $collection->getAttribute('indexes', []);
@@ -2656,10 +2584,6 @@ public function getDocument(string $collection, string $id, array $queries = [])
             throw new DatabaseException('Collection not found');
         }
 
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if (empty($id)) {
             return new Document();
         }
@@ -3085,9 +3009,9 @@ private function populateDocumentRelationships(Document $collection, Document $d
     public function createDocument(string $collection, Document $document): Document
     {
         if (
-            $this->adapter->getSharedTables()
+            $collection !== self::METADATA
+            && $this->adapter->getSharedTables()
             && empty($this->adapter->getTenant())
-            && $collection !== self::METADATA
         ) {
             throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
         }
@@ -3160,10 +3084,6 @@ public function createDocument(string $collection, Document $document): Document
      */
     public function createDocuments(string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE): array
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if (empty($documents)) {
             return [];
         }
@@ -3549,10 +3469,6 @@ private function relateDocumentsById(
      */
     public function updateDocument(string $collection, string $id, Document $document): Document
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if (!$id) {
             throw new DatabaseException('Must define $id attribute');
         }
@@ -3745,10 +3661,6 @@ public function updateDocument(string $collection, string $id, Document $documen
      */
     public function updateDocuments(string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE): array
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if (empty($documents)) {
             return [];
         }
@@ -4218,10 +4130,6 @@ private function getJunctionCollection(Document $collection, Document $relatedCo
      */
     public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $max = null): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if ($value <= 0) { // Can be a float
             throw new DatabaseException('Value must be numeric and greater than 0');
         }
@@ -4308,10 +4216,6 @@ public function increaseDocumentAttribute(string $collection, string $id, string
      */
     public function decreaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $min = null): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         if ($value <= 0) { // Can be a float
             throw new DatabaseException('Value must be numeric and greater than 0');
         }
@@ -4398,10 +4302,6 @@ public function decreaseDocumentAttribute(string $collection, string $id, string
      */
     public function deleteDocument(string $collection, string $id): bool
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $document = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id)));
 
         $collection = $this->silent(fn () => $this->getCollection($collection));
@@ -4876,10 +4776,6 @@ public function purgeCachedDocument(string $collectionId, string $id): bool
      */
     public function find(string $collection, array $queries = []): array
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         if ($collection->isEmpty()) {
@@ -5056,10 +4952,6 @@ public function findOne(string $collection, array $queries = []): false|Document
      */
     public function count(string $collection, array $queries = [], ?int $max = null): int
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
         $attributes = $collection->getAttribute('attributes', []);
         $indexes = $collection->getAttribute('indexes', []);
@@ -5102,10 +4994,6 @@ public function count(string $collection, array $queries = [], ?int $max = null)
      */
     public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null): float|int
     {
-        if ($this->adapter->getSharedTables() && empty($this->adapter->getTenant())) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
         $attributes = $collection->getAttribute('attributes', []);
         $indexes = $collection->getAttribute('indexes', []);

From 4f91b4abceecd048b90f09128e44927b6c0d6fbc Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 21:17:40 +1200
Subject: [PATCH 042/100] Remove get collection check

---
 src/Database/Database.php | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index dd17fb514..d5bdf0ff3 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -1082,14 +1082,6 @@ public function updateCollection(string $id, array $permissions, bool $documentS
      */
     public function getCollection(string $id): Document
     {
-        if (
-            $id !== self::METADATA
-            && $this->adapter->getSharedTables()
-            && empty($this->adapter->getTenant())
-        ) {
-            throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.');
-        }
-
         $collection = $this->silent(fn () => $this->getDocument(self::METADATA, $id));
 
         $tenant = $collection->getAttribute('$tenant');

From 7bc4146a1df044aae127315c2e4ee91bf21720ad Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 21:38:06 +1200
Subject: [PATCH 043/100] Throw on duplicate collection

---
 src/Database/Adapter/MariaDB.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 3506221b7..3b4b6d6c8 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -130,7 +130,7 @@ public function createCollection(string $name, array $attributes = [], array $in
         }
 
         $sql = "
-			CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} (
+			CREATE TABLE {$this->getSQLTable($id)} (
 				_id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT,
 				_uid VARCHAR(255) NOT NULL,
 				_createdAt DATETIME(3) DEFAULT NULL,

From 9a476e82f7be7f83a1e7060be76f90534c396b33 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 22:34:48 +1200
Subject: [PATCH 044/100] Throw duplicate properly, create collections in
 transaction

---
 src/Database/Adapter/MariaDB.php | 68 +++++++++++++++++++-------------
 src/Database/Adapter/MySQL.php   | 11 ++++--
 2 files changed, 48 insertions(+), 31 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 3b4b6d6c8..afb00379a 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -129,7 +129,7 @@ public function createCollection(string $name, array $attributes = [], array $in
             $indexStrings[$key] = "{$indexType} `{$indexId}` ({$indexAttributes}),";
         }
 
-        $sql = "
+        $collectionStmt = "
 			CREATE TABLE {$this->getSQLTable($id)} (
 				_id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT,
 				_uid VARCHAR(255) NOT NULL,
@@ -142,7 +142,7 @@ public function createCollection(string $name, array $attributes = [], array $in
 		";
 
         if ($this->sharedTables) {
-            $sql .= "
+            $collectionStmt .= "
             	_tenant INT(11) UNSIGNED DEFAULT NULL,
 				UNIQUE KEY _uid (_tenant, _uid),
 				KEY _created_at (_tenant, _createdAt),
@@ -150,24 +150,18 @@ public function createCollection(string $name, array $attributes = [], array $in
 				KEY _tenant_id (_tenant, _id)
 			";
         } else {
-            $sql .= "
+            $collectionStmt .= "
 				UNIQUE KEY _uid (_uid),
 				KEY _created_at (_createdAt),
 				KEY _updated_at (_updatedAt)
 			";
         }
 
-        $sql .= ")";
-
-        $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql);
+        $collectionStmt .= ")";
+        $collectionStmt = $this->trigger(Database::EVENT_COLLECTION_CREATE, $collectionStmt);
 
-        try {
-            $this->getPDO()
-                ->prepare($sql)
-                ->execute();
-
-            $sql = "
-				CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} (
+        $permissionsStmt = "
+				CREATE TABLE {$this->getSQLTable($id . '_perms')} (
 					_id int(11) UNSIGNED NOT NULL AUTO_INCREMENT,
 					_type VARCHAR(12) NOT NULL,
 					_permission VARCHAR(255) NOT NULL,
@@ -175,31 +169,46 @@ public function createCollection(string $name, array $attributes = [], array $in
 					PRIMARY KEY (_id),
 			";
 
-            if ($this->sharedTables) {
-                $sql .= "
+        if ($this->sharedTables) {
+            $permissionsStmt .= "
                 	_tenant INT(11) UNSIGNED DEFAULT NULL,
 					UNIQUE INDEX _index1 (_document, _tenant, _type, _permission),
 					INDEX _permission (_tenant, _permission, _type)
 				";
-            } else {
-                $sql .= "
+        } else {
+            $permissionsStmt .= "
 					UNIQUE INDEX _index1 (_document, _type, _permission),
 					INDEX _permission (_permission, _type)
 				";
-            }
+        }
 
-            $sql .= ")";
+        $permissionsStmt .= ")";
+        $permissionsStmt = $this->trigger(Database::EVENT_COLLECTION_CREATE, $permissionsStmt);
 
-            $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql);
+        try {
+            $this->pdo->beginTransaction();
 
             $this->getPDO()
-                ->prepare($sql)
+                ->prepare($collectionStmt)
                 ->execute();
-        } catch (\Exception $th) {
+
             $this->getPDO()
-                ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};")
+                ->prepare($permissionsStmt)
                 ->execute();
-            throw $th;
+
+            if (!$this->pdo->commit()) {
+                throw new DatabaseException('Failed to commit transaction');
+            }
+        } catch (\Exception $e) {
+            if (!$this->pdo->inTransaction()) {
+                $this->pdo->rollBack();
+            }
+
+            if ($e instanceof PDOException) {
+                $this->processException($e);
+            }
+
+            throw $e;
         }
 
         return true;
@@ -2246,17 +2255,20 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
     /**
      * @param PDOException $e
      * @throws TimeoutException
+     * @throws DuplicateException
      */
     protected function processException(PDOException $e): void
     {
-        // Regular PDO
         if ($e->getCode() === '70100' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1969) {
             throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
+        } else if ($e->getCode() === 1969 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '70100') {
+            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
-        // PDOProxy switches errorInfo PDOProxy.php line 64
-        if ($e->getCode() === 1969 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '70100') {
-            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
+        if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
+            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        } else if ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
+            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
         throw $e;
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index 287680390..40ee49d01 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -5,6 +5,7 @@
 use PDOException;
 use Utopia\Database\Database;
 use Utopia\Database\Exception as DatabaseException;
+use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
 
 class MySQL extends MariaDB
@@ -37,16 +38,20 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
     /**
      * @param PDOException $e
      * @throws TimeoutException
+     * @throws DuplicateException
      */
     protected function processException(PDOException $e): void
     {
         if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
             throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
+        } elseif ($e->getCode() === 3024 && isset($e->errorInfo[0]) && $e->errorInfo[0] === "HY000") {
+            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
-        // PDOProxy which who switches errorInfo
-        if ($e->getCode() === 3024 && isset($e->errorInfo[0]) && $e->errorInfo[0] === "HY000") {
-            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
+        if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
+            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        } else if ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
+            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
         throw $e;

From 4dc980c240dfcf503b2a6208ef689dc69e27f79a Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 22:41:53 +1200
Subject: [PATCH 045/100] Use getPDO

---
 src/Database/Adapter/MariaDB.php | 36 ++++++++++++++++----------------
 src/Database/Adapter/MySQL.php   |  2 +-
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index afb00379a..ccece4414 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -129,7 +129,7 @@ public function createCollection(string $name, array $attributes = [], array $in
             $indexStrings[$key] = "{$indexType} `{$indexId}` ({$indexAttributes}),";
         }
 
-        $collectionStmt = "
+        $collection = "
 			CREATE TABLE {$this->getSQLTable($id)} (
 				_id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT,
 				_uid VARCHAR(255) NOT NULL,
@@ -142,7 +142,7 @@ public function createCollection(string $name, array $attributes = [], array $in
 		";
 
         if ($this->sharedTables) {
-            $collectionStmt .= "
+            $collection .= "
             	_tenant INT(11) UNSIGNED DEFAULT NULL,
 				UNIQUE KEY _uid (_tenant, _uid),
 				KEY _created_at (_tenant, _createdAt),
@@ -150,17 +150,17 @@ public function createCollection(string $name, array $attributes = [], array $in
 				KEY _tenant_id (_tenant, _id)
 			";
         } else {
-            $collectionStmt .= "
+            $collection .= "
 				UNIQUE KEY _uid (_uid),
 				KEY _created_at (_createdAt),
 				KEY _updated_at (_updatedAt)
 			";
         }
 
-        $collectionStmt .= ")";
-        $collectionStmt = $this->trigger(Database::EVENT_COLLECTION_CREATE, $collectionStmt);
+        $collection .= ")";
+        $collection = $this->trigger(Database::EVENT_COLLECTION_CREATE, $collection);
 
-        $permissionsStmt = "
+        $permissions = "
 				CREATE TABLE {$this->getSQLTable($id . '_perms')} (
 					_id int(11) UNSIGNED NOT NULL AUTO_INCREMENT,
 					_type VARCHAR(12) NOT NULL,
@@ -170,38 +170,38 @@ public function createCollection(string $name, array $attributes = [], array $in
 			";
 
         if ($this->sharedTables) {
-            $permissionsStmt .= "
+            $permissions .= "
                 	_tenant INT(11) UNSIGNED DEFAULT NULL,
 					UNIQUE INDEX _index1 (_document, _tenant, _type, _permission),
 					INDEX _permission (_tenant, _permission, _type)
 				";
         } else {
-            $permissionsStmt .= "
+            $permissions .= "
 					UNIQUE INDEX _index1 (_document, _type, _permission),
 					INDEX _permission (_permission, _type)
 				";
         }
 
-        $permissionsStmt .= ")";
-        $permissionsStmt = $this->trigger(Database::EVENT_COLLECTION_CREATE, $permissionsStmt);
+        $permissions .= ")";
+        $permissions = $this->trigger(Database::EVENT_COLLECTION_CREATE, $permissions);
 
         try {
-            $this->pdo->beginTransaction();
+            $this->getPDO()->beginTransaction();
 
             $this->getPDO()
-                ->prepare($collectionStmt)
+                ->prepare($collection)
                 ->execute();
 
             $this->getPDO()
-                ->prepare($permissionsStmt)
+                ->prepare($permissions)
                 ->execute();
 
-            if (!$this->pdo->commit()) {
+            if (!$this->getPDO()->commit()) {
                 throw new DatabaseException('Failed to commit transaction');
             }
         } catch (\Exception $e) {
-            if (!$this->pdo->inTransaction()) {
-                $this->pdo->rollBack();
+            if (!$this->getPDO()->inTransaction()) {
+                $this->getPDO()->rollBack();
             }
 
             if ($e instanceof PDOException) {
@@ -2261,13 +2261,13 @@ protected function processException(PDOException $e): void
     {
         if ($e->getCode() === '70100' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1969) {
             throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
-        } else if ($e->getCode() === 1969 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '70100') {
+        } elseif ($e->getCode() === 1969 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '70100') {
             throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
         if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
             throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } else if ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
+        } elseif ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
             throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index 40ee49d01..51c1c627d 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -50,7 +50,7 @@ protected function processException(PDOException $e): void
 
         if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
             throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } else if ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
+        } elseif ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
             throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 

From 88570af189320e8bd6da588203a4c4f3b2999e88 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 22:46:49 +1200
Subject: [PATCH 046/100] Fix check

---
 src/Database/Adapter/MariaDB.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index ccece4414..f32a8c167 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -200,7 +200,7 @@ public function createCollection(string $name, array $attributes = [], array $in
                 throw new DatabaseException('Failed to commit transaction');
             }
         } catch (\Exception $e) {
-            if (!$this->getPDO()->inTransaction()) {
+            if ($this->getPDO()->inTransaction()) {
                 $this->getPDO()->rollBack();
             }
 

From df9ef171c591bdb7491f727f04f3f3806e8e09a8 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 23:25:08 +1200
Subject: [PATCH 047/100] Fix impicit commit

---
 composer.json                    |  2 +-
 composer.lock                    | 36 +++++++++++-----------------
 src/Database/Adapter/MariaDB.php | 40 ++++++++++++++------------------
 3 files changed, 31 insertions(+), 47 deletions(-)

diff --git a/composer.json b/composer.json
index cab94655c..52f8a44f8 100755
--- a/composer.json
+++ b/composer.json
@@ -44,7 +44,7 @@
         "fakerphp/faker": "^1.14",
         "phpunit/phpunit": "^9.4",
         "pcov/clobber": "^2.0",
-        "swoole/ide-helper": "4.8.0",
+        "swoole/ide-helper": "5.1.2",
         "utopia-php/cli": "^0.14.0",
         "laravel/pint": "1.13.*",
         "phpstan/phpstan": "1.10.*",
diff --git a/composer.lock b/composer.lock
index 41ee9ffbf..aca8e7724 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "0b71e4329463ee71c5067fa62792a52f",
+    "content-hash": "1ef7c11d9a0628b3c57c8c80dbebbba1",
     "packages": [
         {
             "name": "jean85/pretty-package-versions",
@@ -266,16 +266,16 @@
         },
         {
             "name": "utopia-php/framework",
-            "version": "0.33.6",
+            "version": "0.33.7",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/http.git",
-                "reference": "8fe57da0cecd57e3b17cd395b4a666a24f4c07a6"
+                "reference": "78d293d99a262bd63ece750bbf989c7e0643b825"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/http/zipball/8fe57da0cecd57e3b17cd395b4a666a24f4c07a6",
-                "reference": "8fe57da0cecd57e3b17cd395b4a666a24f4c07a6",
+                "url": "https://api.github.com/repos/utopia-php/http/zipball/78d293d99a262bd63ece750bbf989c7e0643b825",
+                "reference": "78d293d99a262bd63ece750bbf989c7e0643b825",
                 "shasum": ""
             },
             "require": {
@@ -305,9 +305,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/http/issues",
-                "source": "https://github.com/utopia-php/http/tree/0.33.6"
+                "source": "https://github.com/utopia-php/http/tree/0.33.7"
             },
-            "time": "2024-03-21T18:10:57+00:00"
+            "time": "2024-08-01T14:01:04+00:00"
         },
         {
             "name": "utopia-php/mongo",
@@ -2382,16 +2382,16 @@
         },
         {
             "name": "swoole/ide-helper",
-            "version": "4.8.0",
+            "version": "5.1.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/swoole/ide-helper.git",
-                "reference": "837a2b20242e3cebf0ba1168e876f0f1ca9a14e3"
+                "reference": "33ec7af9111b76d06a70dd31191cc74793551112"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/swoole/ide-helper/zipball/837a2b20242e3cebf0ba1168e876f0f1ca9a14e3",
-                "reference": "837a2b20242e3cebf0ba1168e876f0f1ca9a14e3",
+                "url": "https://api.github.com/repos/swoole/ide-helper/zipball/33ec7af9111b76d06a70dd31191cc74793551112",
+                "reference": "33ec7af9111b76d06a70dd31191cc74793551112",
                 "shasum": ""
             },
             "type": "library",
@@ -2408,19 +2408,9 @@
             "description": "IDE help files for Swoole.",
             "support": {
                 "issues": "https://github.com/swoole/ide-helper/issues",
-                "source": "https://github.com/swoole/ide-helper/tree/4.8.0"
+                "source": "https://github.com/swoole/ide-helper/tree/5.1.2"
             },
-            "funding": [
-                {
-                    "url": "https://gitee.com/swoole/swoole?donate=true",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/swoole",
-                    "type": "github"
-                }
-            ],
-            "time": "2021-10-14T19:39:28+00:00"
+            "time": "2024-02-01T22:28:11+00:00"
         },
         {
             "name": "symfony/deprecation-contracts",
diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index f32a8c167..478bbb7e3 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -161,33 +161,31 @@ public function createCollection(string $name, array $attributes = [], array $in
         $collection = $this->trigger(Database::EVENT_COLLECTION_CREATE, $collection);
 
         $permissions = "
-				CREATE TABLE {$this->getSQLTable($id . '_perms')} (
-					_id int(11) UNSIGNED NOT NULL AUTO_INCREMENT,
-					_type VARCHAR(12) NOT NULL,
-					_permission VARCHAR(255) NOT NULL,
-					_document VARCHAR(255) NOT NULL,
-					PRIMARY KEY (_id),
-			";
+            CREATE TABLE {$this->getSQLTable($id . '_perms')} (
+                _id int(11) UNSIGNED NOT NULL AUTO_INCREMENT,
+                _type VARCHAR(12) NOT NULL,
+                _permission VARCHAR(255) NOT NULL,
+                _document VARCHAR(255) NOT NULL,
+                PRIMARY KEY (_id),
+        ";
 
         if ($this->sharedTables) {
             $permissions .= "
-                	_tenant INT(11) UNSIGNED DEFAULT NULL,
-					UNIQUE INDEX _index1 (_document, _tenant, _type, _permission),
-					INDEX _permission (_tenant, _permission, _type)
-				";
+                _tenant INT(11) UNSIGNED DEFAULT NULL,
+                UNIQUE INDEX _index1 (_document, _tenant, _type, _permission),
+                INDEX _permission (_tenant, _permission, _type)
+            ";
         } else {
             $permissions .= "
-					UNIQUE INDEX _index1 (_document, _type, _permission),
-					INDEX _permission (_permission, _type)
-				";
+                UNIQUE INDEX _index1 (_document, _type, _permission),
+                INDEX _permission (_permission, _type)
+            ";
         }
 
         $permissions .= ")";
         $permissions = $this->trigger(Database::EVENT_COLLECTION_CREATE, $permissions);
 
         try {
-            $this->getPDO()->beginTransaction();
-
             $this->getPDO()
                 ->prepare($collection)
                 ->execute();
@@ -195,14 +193,10 @@ public function createCollection(string $name, array $attributes = [], array $in
             $this->getPDO()
                 ->prepare($permissions)
                 ->execute();
-
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
         } catch (\Exception $e) {
-            if ($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
+            $this->getPDO()
+                ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};")
+                ->execute();
 
             if ($e instanceof PDOException) {
                 $this->processException($e);

From 27444b13495111e6179b14fa53ca55ddf91d192e Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 12 Aug 2024 23:43:30 +1200
Subject: [PATCH 048/100] Update null checks

---
 src/Database/Adapter/MariaDB.php  | 16 ++++++++--------
 src/Database/Adapter/Postgres.php | 16 ++++++++--------
 src/Database/Adapter/SQL.php      |  4 ++--
 src/Database/Adapter/SQLite.php   | 10 +++++-----
 4 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 478bbb7e3..6bc8f3238 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -1049,7 +1049,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
             }
 
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1126,7 +1126,7 @@ public function updateDocument(string $collection, Document $document): Document
                 ";
 
                 if ($this->sharedTables) {
-                    $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                    $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
                 }
 
                 $removeQuery = $sql . $removeQuery;
@@ -1213,7 +1213,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
             }
 
             $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -1342,7 +1342,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                        $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1385,7 +1385,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant IN (:_tenant, NULL)';
+                                $tenantQuery = ' AND _tenant = :_tenant OR _tenant IS NULL';
                             }
 
                             $removeQuery .= "(
@@ -1566,7 +1566,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant IN (:_tenant, NULL)';
+            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
         }
 
         $sql .= $sqlMax . $sqlMin;
@@ -1608,7 +1608,7 @@ public function deleteDocument(string $collection, string $id): bool
 		    ";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
             }
 
             $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
@@ -1627,7 +1627,7 @@ public function deleteDocument(string $collection, string $id): bool
 		    ";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
             }
 
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql);
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index b6e98b069..859a339b0 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -977,7 +977,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant IN (:_tenant, NULL)';
+            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1057,7 +1057,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
             }
 
             $removeQuery = $sql . $removeQuery;
@@ -1129,7 +1129,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant IN (:_tenant, NULL)';
+            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -1253,7 +1253,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= ' AND _tenant IN (:_tenant, NULL)';
+                        $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1296,7 +1296,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant IN (:_tenant, NULL)';
+                                $tenantQuery = ' AND _tenant = :_tenant OR _tenant IS NULL';
                             }
 
                             $removeQuery .= "(
@@ -1467,7 +1467,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant IN (:_tenant, NULL)';
+            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
         }
 
         $sql .= $sqlMax . $sqlMin;
@@ -1507,7 +1507,7 @@ public function deleteDocument(string $collection, string $id): bool
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant IN (:_tenant, NULL)';
+            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
@@ -1524,7 +1524,7 @@ public function deleteDocument(string $collection, string $id): bool
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant IN (:_tenant, NULL)';
+            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql);
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index fdb7d8bea..e50f0ae03 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -116,7 +116,7 @@ public function getDocument(string $collection, string $id, array $queries = [])
 		";
 
         if ($this->sharedTables) {
-            $sql .= "AND _tenant IN (:_tenant, NULL)";
+            $sql .= "AND _tenant = :_tenant OR _tenant IS NULL";
         }
 
         $stmt = $this->getPDO()->prepare($sql);
@@ -862,7 +862,7 @@ protected function getSQLPermissionsCondition(string $collection, array $roles):
 
         $tenantQuery = '';
         if ($this->sharedTables) {
-            $tenantQuery = 'AND _tenant IN (:_tenant, NULL)';
+            $tenantQuery = 'AND _tenant = :_tenant OR _tenant IS NULL';
         }
 
         return "table_main._uid IN (
diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index 115bd879c..16e1bf7cd 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -601,7 +601,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= " AND _tenant IN (:_tenant, NULL)";
+            $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -684,7 +684,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= " AND _tenant IN (:_tenant, NULL)";
+                $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
             }
 
             $removeQuery = $sql . $removeQuery;
@@ -756,7 +756,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= " AND _tenant IN (:_tenant, NULL)";
+            $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -879,7 +879,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= " AND _tenant IN (:_tenant, NULL)";
+                        $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -926,7 +926,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant IN (:_tenant, NULL)';
+                                $tenantQuery = ' AND _tenant = :_tenant OR _tenant IS NULL';
                             }
 
                             $removeQuery .= "(

From ac68fa45dd2fa513241f24723606667b1284d2a4 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 13 Aug 2024 16:04:36 +1200
Subject: [PATCH 049/100] Don't drop collection on duplicate exception

---
 src/Database/Adapter/MariaDB.php  | 34 +++++++--------
 src/Database/Adapter/Mongo.php    | 10 ++---
 src/Database/Adapter/MySQL.php    | 41 ++++++++----------
 src/Database/Adapter/Postgres.php | 72 ++++++++++++++++---------------
 tests/e2e/Adapter/Base.php        | 21 +++++++++
 5 files changed, 97 insertions(+), 81 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 6bc8f3238..0e5e155c6 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -193,13 +193,13 @@ public function createCollection(string $name, array $attributes = [], array $in
             $this->getPDO()
                 ->prepare($permissions)
                 ->execute();
-        } catch (\Exception $e) {
-            $this->getPDO()
-                ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};")
-                ->execute();
+        } catch (PDOException $e) {
+            $e = $this->processException($e);
 
-            if ($e instanceof PDOException) {
-                $this->processException($e);
+            if (!($e instanceof DuplicateException)) {
+                $this->getPDO()
+                    ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};")
+                    ->execute();
             }
 
             throw $e;
@@ -1677,6 +1677,7 @@ public function deleteDocument(string $collection, string $id): bool
      * @return array<Document>
      * @throws DatabaseException
      * @throws TimeoutException
+     * @throws Exception
      */
     public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER): array
     {
@@ -1827,7 +1828,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         try {
             $stmt->execute();
         } catch (PDOException $e) {
-            $this->processException($e);
+            throw $this->processException($e);
         }
 
         $results = $stmt->fetchAll();
@@ -2246,25 +2247,22 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
         });
     }
 
-    /**
-     * @param PDOException $e
-     * @throws TimeoutException
-     * @throws DuplicateException
-     */
-    protected function processException(PDOException $e): void
+    protected function processException(PDOException $e): \Exception
     {
+        // Timeout
         if ($e->getCode() === '70100' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1969) {
-            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
+            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         } elseif ($e->getCode() === 1969 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '70100') {
-            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
+            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
+        // Duplicate table
         if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
-            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         } elseif ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
-            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
-        throw $e;
+        return $e;
     }
 }
diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
index de4da64f9..96e173ab1 100644
--- a/src/Database/Adapter/Mongo.php
+++ b/src/Database/Adapter/Mongo.php
@@ -1755,17 +1755,13 @@ public function getKeywords(): array
         return [];
     }
 
-    /**
-     * @throws Timeout
-     * @throws Exception
-     */
-    protected function processException(Exception $e): void
+    protected function processException(Exception $e): \Exception
     {
         if ($e->getCode() === 50) {
-            throw new Timeout($e->getMessage());
+            return new Timeout($e->getMessage());
         }
 
-        throw $e;
+        return $e;
     }
 
     /**
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index 51c1c627d..ab821db8a 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -35,28 +35,6 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
         });
     }
 
-    /**
-     * @param PDOException $e
-     * @throws TimeoutException
-     * @throws DuplicateException
-     */
-    protected function processException(PDOException $e): void
-    {
-        if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
-            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 3024 && isset($e->errorInfo[0]) && $e->errorInfo[0] === "HY000") {
-            throw new TimeoutException($e->getMessage(), $e->getCode(), $e);
-        }
-
-        if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
-            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
-            throw new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        }
-
-        throw $e;
-    }
-
     /**
     * Get Collection Size
     * @param string $collection
@@ -104,4 +82,23 @@ public function castIndexArray(): bool
     {
         return true;
     }
+
+    protected function processException(PDOException $e): \Exception
+    {
+        // Timeout
+        if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
+            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
+        } elseif ($e->getCode() === 3024 && isset($e->errorInfo[0]) && $e->errorInfo[0] === "HY000") {
+            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
+        }
+
+        // Duplicate table
+        if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        } elseif ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        }
+
+        return $e;
+    }
 }
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index 859a339b0..bff8383b1 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -9,8 +9,8 @@
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Exception as DatabaseException;
-use Utopia\Database\Exception\Duplicate;
-use Utopia\Database\Exception\Timeout;
+use Utopia\Database\Exception\Duplicate as DuplicateException;
+use Utopia\Database\Exception\Timeout as TimeoutException;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 
@@ -827,7 +827,7 @@ public function createDocument(string $collection, Document $document): Document
             switch ($e->getCode()) {
                 case 23505:
                     $this->getPDO()->rollBack();
-                    throw new Duplicate('Duplicated document: ' . $e->getMessage());
+                    throw new DuplicateException('Duplicated document: ' . $e->getMessage());
                 default:
                     throw $e;
             }
@@ -849,7 +849,7 @@ public function createDocument(string $collection, Document $document): Document
      *
      * @return array<Document>
      *
-     * @throws Duplicate
+     * @throws DuplicateException
      */
     public function createDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array
     {
@@ -942,7 +942,7 @@ public function createDocuments(string $collection, array $documents, int $batch
             $this->getPDO()->rollBack();
 
             throw match ($e->getCode()) {
-                1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()),
+                1062, 23000 => new DuplicateException('Duplicated document: ' . $e->getMessage()),
                 default => $e,
             };
         }
@@ -955,6 +955,8 @@ public function createDocuments(string $collection, array $documents, int $batch
      * @param Document $document
      *
      * @return Document
+     * @throws DatabaseException
+     * @throws DuplicateException
      */
     public function updateDocument(string $collection, Document $document): Document
     {
@@ -1164,11 +1166,12 @@ public function updateDocument(string $collection, Document $document): Document
             }
         } catch (PDOException $e) {
             $this->getPDO()->rollBack();
+
+            // Must be a switch for loose match
             switch ($e->getCode()) {
                 case 1062:
                 case 23505:
-                    throw new Duplicate('Duplicated document: ' . $e->getMessage());
-
+                    throw new DuplicateException('Duplicated document: ' . $e->getMessage());
                 default:
                     throw $e;
             }
@@ -1187,10 +1190,9 @@ public function updateDocument(string $collection, Document $document): Document
      * @param string $collection
      * @param array<Document> $documents
      * @param int $batchSize
-     *
      * @return array<Document>
-     *
-     * @throws Duplicate
+     * @throws DatabaseException
+     * @throws DuplicateException
      */
     public function updateDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array
     {
@@ -1430,10 +1432,14 @@ public function updateDocuments(string $collection, array $documents, int $batch
         } catch (PDOException $e) {
             $this->getPDO()->rollBack();
 
-            throw match ($e->getCode()) {
-                1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()),
-                default => $e,
-            };
+            // Must be a switch for loose match
+            switch ($e->getCode()) {
+                case 1062:
+                case 23505:
+                    throw new DuplicateException('Duplicated document: ' . $e->getMessage());
+                default:
+                    throw $e;
+            }
         }
     }
 
@@ -1575,9 +1581,8 @@ public function deleteDocument(string $collection, string $id): bool
      * @param string $cursorDirection
      *
      * @return array<Document>
-     * @throws Exception
-     * @throws PDOException
-     * @throws Timeout
+     * @throws DatabaseException
+     * @throws TimeoutException
      */
     public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER): array
     {
@@ -2195,29 +2200,28 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
     }
 
     /**
-     * @param PDOException $e
-     * @throws Timeout
+     * @return string
      */
-    protected function processException(PDOException $e): void
+    public function getLikeOperator(): string
+    {
+        return 'ILIKE';
+    }
+
+    protected function processException(PDOException $e): \Exception
     {
-        // Regular PDO
+        // Timeout
         if ($e->getCode() === '57014' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
-            throw new Timeout($e->getMessage(), $e->getCode(), $e);
+            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
+        } elseif ($e->getCode() === 7 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '57014') {
+            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
-        // PDOProxy switches errorInfo PDOProxy.php line 64
-        if ($e->getCode() === 7 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '57014') {
-            throw new Timeout($e->getMessage(), $e->getCode(), $e);
+        if ($e->getCode() === '42P07' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        } elseif ($e->getCode() === 7 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42P07') {
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
-        throw $e;
-    }
-
-    /**
-     * @return string
-     */
-    public function getLikeOperator(): string
-    {
-        return 'ILIKE';
+        return $e;
     }
 }
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 662a3ac59..41f90f3de 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -79,6 +79,27 @@ public function testCreateExistsDelete(): void
         $this->assertEquals(true, static::getDatabase()->create());
     }
 
+    /**
+     * @throws LimitException
+     * @throws DuplicateException
+     * @throws DatabaseException
+     */
+    public function testCreateDuplicates(): void
+    {
+        static::getDatabase()->createCollection('test', permissions: [
+            Permission::read(Role::any())
+        ]);
+
+        try {
+            static::getDatabase()->createCollection('test');
+            $this->fail('Failed to throw exception');
+        } catch (Exception $e) {
+            $this->assertInstanceOf(DuplicateException::class, $e);
+        }
+
+        $this->assertNotEmpty(static::getDatabase()->listCollections());
+    }
+
     public function testUpdateDeleteCollectionNotFound(): void
     {
         try {

From d7ae07e0b431c8245a72fb10aea6c08a0290dc4d Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 13 Aug 2024 20:57:53 +1200
Subject: [PATCH 050/100] Fix precedence

---
 src/Database/Adapter/MariaDB.php  | 16 ++++++++--------
 src/Database/Adapter/Postgres.php | 16 ++++++++--------
 src/Database/Adapter/SQL.php      |  4 ++--
 src/Database/Adapter/SQLite.php   | 10 +++++-----
 4 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 0e5e155c6..9738d0ecf 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -1049,7 +1049,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1126,7 +1126,7 @@ public function updateDocument(string $collection, Document $document): Document
                 ";
 
                 if ($this->sharedTables) {
-                    $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                    $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
                 }
 
                 $removeQuery = $sql . $removeQuery;
@@ -1213,7 +1213,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -1342,7 +1342,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                        $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1385,7 +1385,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant = :_tenant OR _tenant IS NULL';
+                                $tenantQuery = ' AND (_tenant = :_tenant OR _tenant IS NULL)';
                             }
 
                             $removeQuery .= "(
@@ -1566,7 +1566,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+            $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
         $sql .= $sqlMax . $sqlMin;
@@ -1608,7 +1608,7 @@ public function deleteDocument(string $collection, string $id): bool
 		    ";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
@@ -1627,7 +1627,7 @@ public function deleteDocument(string $collection, string $id): bool
 		    ";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
             }
 
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql);
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index bff8383b1..7196fbdfc 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -979,7 +979,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+            $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1059,7 +1059,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
             }
 
             $removeQuery = $sql . $removeQuery;
@@ -1131,7 +1131,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+            $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -1255,7 +1255,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+                        $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -1298,7 +1298,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant = :_tenant OR _tenant IS NULL';
+                                $tenantQuery = ' AND (_tenant = :_tenant OR _tenant IS NULL)';
                             }
 
                             $removeQuery .= "(
@@ -1473,7 +1473,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+            $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
         $sql .= $sqlMax . $sqlMin;
@@ -1513,7 +1513,7 @@ public function deleteDocument(string $collection, string $id): bool
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+            $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql);
@@ -1530,7 +1530,7 @@ public function deleteDocument(string $collection, string $id): bool
 		";
 
         if ($this->sharedTables) {
-            $sql .= ' AND _tenant = :_tenant OR _tenant IS NULL';
+            $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql);
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index e50f0ae03..2b76380f8 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -116,7 +116,7 @@ public function getDocument(string $collection, string $id, array $queries = [])
 		";
 
         if ($this->sharedTables) {
-            $sql .= "AND _tenant = :_tenant OR _tenant IS NULL";
+            $sql .= "AND (_tenant = :_tenant OR _tenant IS NULL)";
         }
 
         $stmt = $this->getPDO()->prepare($sql);
@@ -862,7 +862,7 @@ protected function getSQLPermissionsCondition(string $collection, array $roles):
 
         $tenantQuery = '';
         if ($this->sharedTables) {
-            $tenantQuery = 'AND _tenant = :_tenant OR _tenant IS NULL';
+            $tenantQuery = 'AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
         return "table_main._uid IN (
diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index 16e1bf7cd..acadb025f 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -601,7 +601,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
+            $sql .= " AND (_tenant = :_tenant OR _tenant IS NULL)";
         }
 
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -684,7 +684,7 @@ public function updateDocument(string $collection, Document $document): Document
 			";
 
             if ($this->sharedTables) {
-                $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
+                $sql .= " AND (_tenant = :_tenant OR _tenant IS NULL)";
             }
 
             $removeQuery = $sql . $removeQuery;
@@ -756,7 +756,7 @@ public function updateDocument(string $collection, Document $document): Document
 		";
 
         if ($this->sharedTables) {
-            $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
+            $sql .= " AND (_tenant = :_tenant OR _tenant IS NULL)";
         }
 
         $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql);
@@ -879,7 +879,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     ";
 
                     if ($this->sharedTables) {
-                        $sql .= " AND _tenant = :_tenant OR _tenant IS NULL";
+                        $sql .= " AND (_tenant = :_tenant OR _tenant IS NULL)";
                     }
 
                     $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
@@ -926,7 +926,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
                             $tenantQuery = '';
                             if ($this->sharedTables) {
-                                $tenantQuery = ' AND _tenant = :_tenant OR _tenant IS NULL';
+                                $tenantQuery = ' AND (_tenant = :_tenant OR _tenant IS NULL)';
                             }
 
                             $removeQuery .= "(

From 04ffb860d85054b2d2bc18e9ad2a7d19d8719c13 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 13 Aug 2024 23:01:34 +1200
Subject: [PATCH 051/100] Allow getTenant

---
 src/Database/Database.php | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 4cafdc24d..9dcee89ac 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -825,6 +825,18 @@ public function setTenant(?int $tenant): static
         return $this;
     }
 
+    /**
+     * Get Tenant
+     *
+     * Get tenant to use if tables are shared
+     *
+     * @return ?int
+     */
+    public function getTenant(): ?int
+    {
+        return $this->adapter->getTenant();
+    }
+
     public function setPreserveDates(bool $preserve): static
     {
         $this->preserveDates = $preserve;

From e37b23fb2eeba1e49a12ddca8438dacf23105219 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 14 Aug 2024 13:05:13 +1200
Subject: [PATCH 052/100] Re-order properties

---
 src/Database/Database.php | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 08f872065..321d3f5e5 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -136,17 +136,6 @@ class Database
 
     public const INSERT_BATCH_SIZE = 100;
 
-    protected Adapter $adapter;
-
-    protected Cache $cache;
-
-    protected string $cacheName = 'default';
-
-    /**
-     * @var array<bool|string>
-     */
-    protected array $map = [];
-
     /**
      * List of Internal attributes
      *
@@ -288,6 +277,17 @@ class Database
         'indexes' => [],
     ];
 
+    protected Adapter $adapter;
+
+    protected Cache $cache;
+
+    protected string $cacheName = 'default';
+
+    /**
+     * @var array<bool|string>
+     */
+    protected array $map = [];
+
     /**
      * @var array<string, array{encode: callable, decode: callable}>
      */

From 545192b81a1ee1d6675c7d197537bf0d0891c3d3 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 15 Aug 2024 20:47:43 +1200
Subject: [PATCH 053/100] Add withTenant, withPreservedDates

---
 src/Database/Database.php | 130 ++++++++++++++++++++++++++++----------
 1 file changed, 97 insertions(+), 33 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 321d3f5e5..556c7837a 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -299,14 +299,14 @@ class Database
     protected array $instanceFilters = [];
 
     /**
-     * @var array<string, mixed>
+     * @var array<string, array<string, callable>>
      */
     protected array $listeners = [
         '*' => [],
     ];
 
     /**
-     * Array in which the keys are the names of databse listeners that
+     * Array in which the keys are the names of database listeners that
      * should be skipped when dispatching events. null $silentListeners
      * will skip all listeners.
      *
@@ -347,8 +347,11 @@ class Database
      * @param Cache $cache
      * @param array<string, array{encode: callable, decode: callable}> $filters
      */
-    public function __construct(Adapter $adapter, Cache $cache, array $filters = [])
-    {
+    public function __construct(
+        Adapter $adapter,
+        Cache $cache,
+        array $filters = []
+    ) {
         $this->adapter = $adapter;
         $this->cache = $cache;
         $this->instanceFilters = $filters;
@@ -724,7 +727,6 @@ public function clearTimeout(string $event = Database::EVENT_ALL): void
     public function enableFilters(): static
     {
         $this->filter = true;
-
         return $this;
     }
 
@@ -736,10 +738,30 @@ public function enableFilters(): static
     public function disableFilters(): static
     {
         $this->filter = false;
-
         return $this;
     }
 
+    /**
+     * Skip filters
+     *
+     * Execute a callback without filters
+     *
+     * @template T
+     * @param callable(): T $callback
+     * @return T
+     */
+    public function skipFilters(callable $callback): mixed
+    {
+        $initial = $this->filter;
+        $this->disableFilters();
+
+        try {
+            return $callback();
+        } finally {
+            $this->filter = $initial;
+        }
+    }
+
     /**
      * Get instance filters
      *
@@ -777,7 +799,7 @@ public function disableValidation(): static
     /**
      * Skip Validation
      *
-     * Skips validation for the code to be executed inside the callback
+     * Execute a callback without validation
      *
      * @template T
      * @param callable(): T $callback
@@ -796,7 +818,18 @@ public function skipValidation(callable $callback): mixed
     }
 
     /**
-     * Set Share Tables
+     * Get shared tables
+     *
+     * Get whether to share tables between tenants
+     * @return bool
+     */
+    public function getSharedTables(): bool
+    {
+        return $this->adapter->getSharedTables();
+    }
+
+    /**
+     * Set shard tables
      *
      * Set whether to share tables between tenants
      *
@@ -837,6 +870,32 @@ public function getTenant(): ?int
         return $this->adapter->getTenant();
     }
 
+    /**
+     * With Tenant
+     *
+     * Execute a callback with a specific tenant
+     *
+     * @param int|null $tenant
+     * @param callable $callback
+     * @return mixed
+     */
+    public function withTenant(?int $tenant, callable $callback): mixed
+    {
+        $previous = $this->adapter->getTenant();
+        $this->adapter->setTenant($tenant);
+
+        try {
+            return $callback();
+        } finally {
+            $this->adapter->setTenant($previous);
+        }
+    }
+
+    public function getPreserveDates(): bool
+    {
+        return $this->preserveDates;
+    }
+
     public function setPreserveDates(bool $preserve): static
     {
         $this->preserveDates = $preserve;
@@ -844,6 +903,18 @@ public function setPreserveDates(bool $preserve): static
         return $this;
     }
 
+    public function withPreservedDates(bool $preserve, callable $callback): mixed
+    {
+        $previous = $this->preserveDates;
+        $this->preserveDates = $preserve;
+
+        try {
+            return $callback();
+        } finally {
+            $this->preserveDates = $previous;
+        }
+    }
+
     /**
      * Get list of keywords that cannot be used
      *
@@ -877,9 +948,11 @@ public function ping(): bool
     /**
      * Create the database
      *
-     * @throws DatabaseException
-     *
+     * @param string|null $database
      * @return bool
+     * @throws DuplicateException
+     * @throws LimitException
+     * @throws Exception
      */
     public function create(?string $database = null): bool
     {
@@ -1129,13 +1202,6 @@ public function listCollections(int $limit = 25, int $offset = 0): array
             Query::offset($offset)
         ]));
 
-        // TODO: Should this be required?
-        //if ($this->adapter->getSharedTables()) {
-        //    $result = \array_filter($result, function ($collection) {
-        //        return $collection->getAttribute('$tenant') == $this->adapter->getTenant();
-        //    });
-        //}
-
         $this->trigger(self::EVENT_COLLECTION_LIST, $result);
 
         return $result;
@@ -1147,6 +1213,7 @@ public function listCollections(int $limit = 25, int $offset = 0): array
      * @param string $collection
      *
      * @return int
+     * @throws Exception
      */
     public function getSizeOfCollection(string $collection): int
     {
@@ -1230,6 +1297,7 @@ public function deleteCollection(string $id): bool
      * @throws DuplicateException
      * @throws LimitException
      * @throws StructureException
+     * @throws Exception
      */
     public function createAttribute(string $collection, string $id, string $type, int $size, bool $required, mixed $default = null, bool $signed = true, bool $array = false, string $format = null, array $formatOptions = [], array $filters = []): bool
     {
@@ -1239,11 +1307,7 @@ public function createAttribute(string $collection, string $id, string $type, in
             throw new DatabaseException('Collection not found');
         }
 
-        if ($this->adapter->getSharedTables() && $collection->getAttribute('$tenant') != $this->adapter->getTenant()) {
-            throw new DatabaseException('Collection not found');
-        }
-
-        // Attribute IDs are case insensitive
+        // Attribute IDs are case-insensitive
         $attributes = $collection->getAttribute('attributes', []);
         /** @var array<Document> $attributes */
         foreach ($attributes as $attribute) {
@@ -1252,9 +1316,9 @@ public function createAttribute(string $collection, string $id, string $type, in
             }
         }
 
-        /** Ensure required filters for the attribute are passed */
+        // Ensure required filters for the attribute are passed
         $requiredFilters = $this->getRequiredFilters($type);
-        if (!empty(array_diff($requiredFilters, $filters))) {
+        if (!empty(\array_diff($requiredFilters, $filters))) {
             throw new DatabaseException("Attribute of type: $type requires the following filters: " . implode(",", $requiredFilters));
         }
 
@@ -1316,7 +1380,7 @@ public function createAttribute(string $collection, string $id, string $type, in
                 throw new DatabaseException('Unknown attribute type: ' . $type . '. Must be one of ' . self::VAR_STRING . ', ' . self::VAR_INTEGER .  ', ' . self::VAR_FLOAT . ', ' . self::VAR_BOOLEAN . ', ' . self::VAR_DATETIME . ', ' . self::VAR_RELATIONSHIP);
         }
 
-        // only execute when $default is given
+        // Only execute when $default is given
         if (!\is_null($default)) {
             if ($required === true) {
                 throw new DatabaseException('Cannot set a default value on a required attribute');
@@ -1364,7 +1428,7 @@ protected function getRequiredFilters(?string $type): array
      * @param string $type Type of the attribute
      * @param mixed $default Default value of the attribute
      *
-     * @throws Exception
+     * @throws DatabaseException
      * @return void
      */
     protected function validateDefaultTypes(string $type, mixed $default): void
@@ -2724,7 +2788,6 @@ public function getDocument(string $collection, string $id, array $queries = [])
 
         /**
          * Bug with function purity in PHPStan means it thinks $this->map is always empty
-         *
          * @phpstan-ignore-next-line
          */
         foreach ($this->map as $key => $value) {
@@ -2740,10 +2803,10 @@ public function getDocument(string $collection, string $id, array $queries = [])
             }
         }
 
-        // Don't save to cache if it's part of a two-way relationship or a relationship at all
+        // Don't save to cache if it's part of a relationship
         if (!$hasTwoWayRelationship && empty($relationships)) {
             $this->cache->save($documentCacheKey, $document->getArrayCopy(), $documentCacheHash);
-            //add document reference to the collection key
+            // Add document reference to the collection key
             $this->cache->save($collectionCacheKey, 'empty', $documentCacheKey);
         }
 
@@ -2754,7 +2817,7 @@ public function getDocument(string $collection, string $id, array $queries = [])
             if ($query->getMethod() === Query::TYPE_SELECT) {
                 $values = $query->getValues();
                 foreach ($this->getInternalAttributes() as $internalAttribute) {
-                    if (!in_array($internalAttribute['$id'], $values)) {
+                    if (!\in_array($internalAttribute['$id'], $values)) {
                         $document->removeAttribute($internalAttribute['$id']);
                     }
                 }
@@ -4782,6 +4845,7 @@ public function purgeCachedDocument(string $collectionId, string $id): bool
      * @throws DatabaseException
      * @throws QueryException
      * @throws TimeoutException
+     * @throws Exception
      */
     public function find(string $collection, array $queries = []): array
     {
@@ -5067,12 +5131,12 @@ public function encode(Document $collection, Document $document): Document
             $filters = $attribute['filters'] ?? [];
             $value = $document->getAttribute($key);
 
-            // continue on optional param with no default
+            // Continue on optional param with no default
             if (is_null($value) && is_null($default)) {
                 continue;
             }
 
-            // assign default only if no value provided
+            // Assign default only if no value provided
             // False positive "Call to function is_null() with mixed will always evaluate to false"
             // @phpstan-ignore-next-line
             if (is_null($value) && !is_null($default)) {
@@ -5256,7 +5320,7 @@ protected function encodeAttribute(string $name, mixed $value, Document $documen
         }
 
         try {
-            if (array_key_exists($name, $this->instanceFilters)) {
+            if (\array_key_exists($name, $this->instanceFilters)) {
                 $value = $this->instanceFilters[$name]['encode']($value, $document, $this);
             } else {
                 $value = self::$filters[$name]['encode']($value, $document, $this);

From 0682d0b1e14910008cceeb6d38dfd8187d31aed7 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 15 Aug 2024 21:30:07 +1200
Subject: [PATCH 054/100] Always preverse in withPreserveDates

---
 src/Database/Database.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 556c7837a..0eb012abf 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -903,10 +903,10 @@ public function setPreserveDates(bool $preserve): static
         return $this;
     }
 
-    public function withPreservedDates(bool $preserve, callable $callback): mixed
+    public function withPreserveDates(callable $callback): mixed
     {
         $previous = $this->preserveDates;
-        $this->preserveDates = $preserve;
+        $this->preserveDates = true;
 
         try {
             return $callback();

From 6662a2cc5205dac0f71916679be615935972f4dc Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 16 Aug 2024 20:18:30 +1200
Subject: [PATCH 055/100] Fix tests

---
 tests/e2e/Adapter/Base.php | 53 ++++----------------------------------
 1 file changed, 5 insertions(+), 48 deletions(-)

diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index bc5a6ace2..7223103d5 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -85,18 +85,20 @@ public function testCreateExistsDelete(): void
      */
     public function testCreateDuplicates(): void
     {
-        static::getDatabase()->createCollection('test', permissions: [
+        static::getDatabase()->createCollection('duplicates', permissions: [
             Permission::read(Role::any())
         ]);
 
         try {
-            static::getDatabase()->createCollection('test');
+            static::getDatabase()->createCollection('duplicates');
             $this->fail('Failed to throw exception');
         } catch (Exception $e) {
             $this->assertInstanceOf(DuplicateException::class, $e);
         }
 
         $this->assertNotEmpty(static::getDatabase()->listCollections());
+
+        static::getDatabase()->deleteCollection('duplicates');
     }
 
     public function testUpdateDeleteCollectionNotFound(): void
@@ -15206,7 +15208,7 @@ public function testSharedTables(): void
             $database->getDocument('people', $docId);
             $this->fail('Failed to throw exception');
         } catch (Exception $e) {
-            $this->assertEquals('Missing tenant. Tenant must be set when table sharing is enabled.', $e->getMessage());
+            $this->assertEquals('Collection not found', $e->getMessage());
         }
 
         // Reset state
@@ -15215,51 +15217,6 @@ public function testSharedTables(): void
         $database->setDatabase($this->testDatabase);
     }
 
-    public function testSharedTablesDuplicateAttributesDontThrow(): void
-    {
-        $database = static::getDatabase();
-
-        if (!$database->getAdapter()->getSupportForAttributes()) {
-            $this->expectNotToPerformAssertions();
-            return;
-        }
-
-        if ($database->exists('sharedTables')) {
-            $database->setDatabase('sharedTables')->delete();
-        }
-
-        $database
-            ->setDatabase('sharedTables')
-            ->setNamespace('')
-            ->setSharedTables(true)
-            ->setTenant(1)
-            ->create();
-
-        // Create collection
-        $database->createCollection('duplicates', documentSecurity: false);
-        $database->createAttribute('duplicates', 'name', Database::VAR_STRING, 10, false);
-
-        $database->setTenant(2);
-        try {
-            $database->createCollection('duplicates', documentSecurity: false);
-        } catch (DuplicateException) {
-            // Ignore
-        }
-        $database->createAttribute('duplicates', 'name', Database::VAR_STRING, 10, false);
-
-        $collection = $database->getCollection('duplicates');
-        $this->assertEquals(1, \count($collection->getAttribute('attributes')));
-
-        $database->setTenant(1);
-
-        $collection = $database->getCollection('duplicates');
-        $this->assertEquals(1, \count($collection->getAttribute('attributes')));
-
-        $database->setSharedTables(false);
-        $database->setNamespace(static::$namespace);
-        $database->setDatabase($this->testDatabase);
-    }
-
     public function testTransformations(): void
     {
         static::getDatabase()->createCollection('docs', attributes: [

From da8113e0514fdaeedd6b13e36b3f02d547f7dfe0 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 15:22:39 +1200
Subject: [PATCH 056/100] Fix in cases

---
 src/Database/Adapter/MariaDB.php  |   8 +-
 src/Database/Adapter/Postgres.php | 137 +++++++++++++++++++-----------
 tests/e2e/Adapter/Base.php        |   8 +-
 3 files changed, 93 insertions(+), 60 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index fcaf38c6c..7aed62094 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -923,10 +923,8 @@ public function createDocuments(string $collection, array $documents, int $batch
 
         try {
             $this->getPDO()->beginTransaction();
-
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, \max(1, $batchSize));
-
             $internalIds = [];
 
             foreach ($batches as $batch) {
@@ -1812,7 +1810,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant IN (:_tenant, NULL)";
+            $where[] = "(table_main._tenant = :_tenant OR table_main._tenant IS NULL)";
         }
 
         $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : '';
@@ -1938,7 +1936,7 @@ public function count(string $collection, array $queries = [], ?int $max = null)
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant IN (:_tenant, NULL)";
+            $where[] = "(table_main._tenant = :_tenant OR table_main._tenant IS NULL)";
         }
 
         $sqlWhere = !empty($where)
@@ -2008,7 +2006,7 @@ public function sum(string $collection, string $attribute, array $queries = [],
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant IN (:_tenant, NULL)";
+            $where[] = "(table_main._tenant = :_tenant OR table_main._tenant IS NULL)";
         }
 
         $sqlWhere = !empty($where)
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index cc8261dae..a6d4a9b7b 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -76,6 +76,7 @@ public function delete(string $name): bool
      * @param array<Document> $attributes
      * @param array<Document> $indexes
      * @return bool
+     * @throws DuplicateException
      */
     public function createCollection(string $name, array $attributes = [], array $indexes = []): bool
     {
@@ -85,8 +86,6 @@ public function createCollection(string $name, array $attributes = [], array $in
         /** @var array<string> $attributeStrings */
         $attributeStrings = [];
 
-        $this->getPDO()->beginTransaction();
-
         /** @var array<string> $attributeStrings */
         $attributeStrings = [];
         foreach ($attributes as $attribute) {
@@ -99,13 +98,30 @@ public function createCollection(string $name, array $attributes = [], array $in
                 $attribute->getAttribute('array', false)
             );
 
+            // Ignore relationships with virtual attributes
+            if ($attribute->getAttribute('type') === Database::VAR_RELATIONSHIP) {
+                $options = $attribute->getAttribute('options', []);
+                $relationType = $options['relationType'] ?? null;
+                $twoWay = $options['twoWay'] ?? false;
+                $side = $options['side'] ?? null;
+
+                if (
+                    $relationType === Database::RELATION_MANY_TO_MANY
+                    || ($relationType === Database::RELATION_ONE_TO_ONE && !$twoWay && $side === Database::RELATION_SIDE_CHILD)
+                    || ($relationType === Database::RELATION_ONE_TO_MANY && $side === Database::RELATION_SIDE_PARENT)
+                    || ($relationType === Database::RELATION_MANY_TO_ONE && $side === Database::RELATION_SIDE_CHILD)
+                ) {
+                    continue;
+                }
+            }
+
             $attributeStrings[] = "\"{$attrId}\" {$attrType}, ";
         }
 
         $sqlTenant = $this->sharedTables ? '_tenant INTEGER DEFAULT NULL,' : '';
 
-        $sql = "
-            CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} (
+        $collection = "
+            CREATE TABLE {$this->getSQLTable($id)} (
                 _id SERIAL NOT NULL,
                 _uid VARCHAR(255) NOT NULL,
                 ". $sqlTenant ."
@@ -116,59 +132,60 @@ public function createCollection(string $name, array $attributes = [], array $in
                 PRIMARY KEY (_id)
             );
         ";
+
         if ($this->sharedTables) {
-            $sql .= "
+            $collection .= "
 				CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid), _tenant);
             	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\");
             	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\");
             	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_tenant_id\" ON {$this->getSQLTable($id)} (_tenant, _id);
 			";
         } else {
-            $sql .= "
+            $collection .= "
 				CREATE UNIQUE INDEX \"{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid));
             	CREATE INDEX \"{$namespace}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\");
             	CREATE INDEX \"{$namespace}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\");
 			";
         }
 
-        $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql);
-
-        $stmt = $this->getPDO()->prepare($sql);
+        $collection = $this->trigger(Database::EVENT_COLLECTION_CREATE, $collection);
 
-        try {
-            $stmt->execute();
+        $permissions = "
+            CREATE TABLE {$this->getSQLTable($id . '_perms')} (
+                _id SERIAL NOT NULL,
+                _tenant INTEGER DEFAULT NULL,
+                _type VARCHAR(12) NOT NULL,
+                _permission VARCHAR(255) NOT NULL,
+                _document VARCHAR(255) NOT NULL,
+                PRIMARY KEY (_id)
+            );   
+        ";
 
-            $sql = "
-				CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} (
-					_id SERIAL NOT NULL,
-					_tenant INTEGER DEFAULT NULL,
-					_type VARCHAR(12) NOT NULL,
-					_permission VARCHAR(255) NOT NULL,
-					_document VARCHAR(255) NOT NULL,
-					PRIMARY KEY (_id)
-				);   
-			";
+        if ($this->sharedTables) {
+            $permissions .= "
+                CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_ukey\" 
+                    ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_document,_type,_permission);
+                CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_permission\" 
+                    ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_permission,_type); 
+            ";
+        } else {
+            $permissions .= "
+                CREATE UNIQUE INDEX \"{$namespace}_{$id}_ukey\" 
+                    ON {$this->getSQLTable($id. '_perms')} USING btree (_document,_type,_permission);
+                CREATE INDEX \"{$namespace}_{$id}_permission\" 
+                    ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type); 
+            ";
+        }
 
-            if ($this->sharedTables) {
-                $sql .= "
-					CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_ukey\" 
-				    	ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_document,_type,_permission);
-					CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_permission\" 
-				    	ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_permission,_type); 
-				";
-            } else {
-                $sql .= "
-					CREATE UNIQUE INDEX \"{$namespace}_{$id}_ukey\" 
-				    	ON {$this->getSQLTable($id. '_perms')} USING btree (_document,_type,_permission);
-					CREATE INDEX \"{$namespace}_{$id}_permission\" 
-				    	ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type); 
-				";
-            }
+        $permissions = $this->trigger(Database::EVENT_COLLECTION_CREATE, $permissions);
 
-            $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql);
+        try {
+            $this->getPDO()
+                ->prepare($collection)
+                ->execute();
 
             $this->getPDO()
-                ->prepare($sql)
+                ->prepare($permissions)
                 ->execute();
 
             foreach ($indexes as $index) {
@@ -186,13 +203,16 @@ public function createCollection(string $name, array $attributes = [], array $in
                     $indexOrders
                 );
             }
-        } catch (Exception $e) {
-            $this->getPDO()->rollBack();
-            throw new DatabaseException('Failed to create collection: ' . $e->getMessage());
-        }
+        } catch (PDOException $e) {
+            $e = $this->processException($e);
 
-        if (!$this->getPDO()->commit()) {
-            throw new DatabaseException('Failed to commit transaction');
+            if (!($e instanceof DuplicateException)) {
+                $this->getPDO()
+                    ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};")
+                    ->execute();
+            }
+
+            throw $e;
         }
 
         return true;
@@ -900,11 +920,12 @@ public function createDocuments(string $collection, array $documents, int $batch
             return $documents;
         }
 
-        $this->getPDO()->beginTransaction();
 
         try {
+            $this->getPDO()->beginTransaction();
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, max(1, $batchSize));
+            $internalIds = [];
 
             foreach ($batches as $batch) {
                 $bindIndex = 0;
@@ -919,6 +940,11 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $attributes['_updatedAt'] = $document->getUpdatedAt();
                     $attributes['_permissions'] = \json_encode($document->getPermissions());
 
+                    if(!empty($document->getInternalId())) {
+                        $internalIds[$document->getId()] = true;
+                        $attributes['_id'] = $document->getInternalId();
+                    }
+
                     if($this->sharedTables) {
                         $attributes['_tenant'] = $this->tenant;
                     }
@@ -978,9 +1004,6 @@ public function createDocuments(string $collection, array $documents, int $batch
             if (!$this->getPDO()->commit()) {
                 throw new DatabaseException('Failed to commit transaction');
             }
-
-            return $documents;
-
         } catch (PDOException $e) {
             $this->getPDO()->rollBack();
 
@@ -989,6 +1012,18 @@ public function createDocuments(string $collection, array $documents, int $batch
                 default => $e,
             };
         }
+
+        foreach ($documents as $document) {
+            if(!isset($internalIds[$document->getId()])) {
+                $document['$internalId'] = $this->getDocument(
+                    $collection,
+                    $document->getId(),
+                    [Query::select(['$internalId'])]
+                )->getInternalId();
+            }
+        }
+
+        return $documents;
     }
 
     /**
@@ -1709,7 +1744,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant IN (:_tenant, NULL)";
+            $where[] = "(table_main._tenant = :_tenant OR table_main._tenant IS NULL)";
         }
 
         if (Authorization::$status) {
@@ -1835,7 +1870,7 @@ public function count(string $collection, array $queries = [], ?int $max = null)
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant IN (:_tenant, NULL)";
+            $where[] = "(table_main._tenant = :_tenant OR table_main._tenant IS NULL)";
         }
 
         if (Authorization::$status) {
@@ -1898,7 +1933,7 @@ public function sum(string $collection, string $attribute, array $queries = [],
         }
 
         if ($this->sharedTables) {
-            $where[] = "table_main._tenant IN (:_tenant, NULL)";
+            $where[] = "(table_main._tenant = :_tenant OR table_main._tenant IS NULL)";
         }
 
         if (Authorization::$status) {
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index e1bb58beb..3ec041afe 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -1001,18 +1001,18 @@ public function testQueryTimeout(): void
                 ]));
             }
 
-            $this->expectException(TimeoutException::class);
-
             static::getDatabase()->setTimeout(1);
 
             try {
                 static::getDatabase()->find('global-timeouts', [
                     Query::notEqual('longtext', 'appwrite'),
                 ]);
-            } catch(TimeoutException $ex) {
+                $this->fail('Failed to throw exception');
+            } catch(\Exception $e) {
+                \var_dump($e);
+                $this->assertInstanceOf(TimeoutException::class, $e);
                 static::getDatabase()->clearTimeout();
                 static::getDatabase()->deleteCollection('global-timeouts');
-                throw $ex;
             }
         }
 

From 9f765890c403c8300c917a2248ce416a659971a6 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 18:24:27 +1200
Subject: [PATCH 057/100] Update timeout test

---
 tests/e2e/Adapter/Base.php | 52 ++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 27 deletions(-)

diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 3ec041afe..a2aea9b30 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -986,37 +986,35 @@ public function testCreatedAtUpdatedAt(): void
 
     public function testQueryTimeout(): void
     {
-        if ($this->getDatabase()->getAdapter()->getSupportForTimeouts()) {
-            static::getDatabase()->createCollection('global-timeouts');
-            $this->assertEquals(true, static::getDatabase()->createAttribute('global-timeouts', 'longtext', Database::VAR_STRING, 100000000, true));
+        if (!$this->getDatabase()->getAdapter()->getSupportForTimeouts()) {
+            $this->expectNotToPerformAssertions();
+        }
+        static::getDatabase()->createCollection('global-timeouts');
+        $this->assertEquals(true, static::getDatabase()->createAttribute('global-timeouts', 'longtext', Database::VAR_STRING, 100000000, true));
 
-            for ($i = 0 ; $i <= 20 ; $i++) {
-                static::getDatabase()->createDocument('global-timeouts', new Document([
-                    'longtext' => file_get_contents(__DIR__ . '/../../resources/longtext.txt'),
-                    '$permissions' => [
-                        Permission::read(Role::any()),
-                        Permission::update(Role::any()),
-                        Permission::delete(Role::any())
-                    ]
-                ]));
-            }
+        for ($i = 0 ; $i <= 200 ; $i++) {
+            static::getDatabase()->createDocument('global-timeouts', new Document([
+                'longtext' => file_get_contents(__DIR__ . '/../../resources/longtext.txt'),
+                '$permissions' => [
+                    Permission::read(Role::any()),
+                    Permission::update(Role::any()),
+                    Permission::delete(Role::any())
+                ]
+            ]));
+        }
 
-            static::getDatabase()->setTimeout(1);
+        static::getDatabase()->setTimeout(1);
 
-            try {
-                static::getDatabase()->find('global-timeouts', [
-                    Query::notEqual('longtext', 'appwrite'),
-                ]);
-                $this->fail('Failed to throw exception');
-            } catch(\Exception $e) {
-                \var_dump($e);
-                $this->assertInstanceOf(TimeoutException::class, $e);
-                static::getDatabase()->clearTimeout();
-                static::getDatabase()->deleteCollection('global-timeouts');
-            }
+        try {
+            static::getDatabase()->find('global-timeouts', [
+                Query::notEqual('longtext', 'appwrite'),
+            ]);
+            $this->fail('Failed to throw exception');
+        } catch(\Exception $e) {
+            static::getDatabase()->clearTimeout();
+            static::getDatabase()->deleteCollection('global-timeouts');
+            $this->assertInstanceOf(TimeoutException::class, $e);
         }
-
-        $this->expectNotToPerformAssertions();
     }
 
     /**

From 0ba61b1db5ac28c63a3154ac3a37eb4174282a8b Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 18:25:20 +1200
Subject: [PATCH 058/100] Fix dump output truncation

---
 dev/xdebug.ini | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dev/xdebug.ini b/dev/xdebug.ini
index ddd00b1a2..34ec0e4b6 100644
--- a/dev/xdebug.ini
+++ b/dev/xdebug.ini
@@ -7,3 +7,7 @@ xdebug.client_host=host.docker.internal
 xdebug.client_port = 9003
 xdebug.log = /tmp/xdebug.log
 xdebug.use_compression=false
+
+xdebug.var_display_max_depth = 10
+xdebug.var_display_max_children = 256
+xdebug.var_display_max_data = 4096

From ab549c157bbfadbfad0ab1144fd0ea3953f30170 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 18:26:02 +1200
Subject: [PATCH 059/100] Fix timeouts not thrown

---
 src/Database/Adapter/MariaDB.php  |  3 +--
 src/Database/Adapter/Mongo.php    |  2 +-
 src/Database/Adapter/Postgres.php |  6 +++---
 tests/e2e/Adapter/Base.php        | 15 +++++++++++++--
 4 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 7aed62094..e85cd236c 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -314,8 +314,7 @@ public function createAttribute(string $collection, string $id, string $type, in
                 ->prepare($sql)
                 ->execute();
         } catch (PDOException $e) {
-            $this->processException($e);
-            return false;
+            throw $this->processException($e);
         }
     }
 
diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
index 8a8706a58..ba84b942a 100644
--- a/src/Database/Adapter/Mongo.php
+++ b/src/Database/Adapter/Mongo.php
@@ -1059,7 +1059,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         try {
             $results = $this->client->find($name, $filters, $options)->cursor->firstBatch ?? [];
         } catch (MongoException $e) {
-            $this->processException($e);
+            throw $this->processException($e);
         }
 
         if (empty($results)) {
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index a6d4a9b7b..bb75b23c5 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -300,8 +300,7 @@ public function createAttribute(string $collection, string $id, string $type, in
                 ->prepare($sql)
                 ->execute();
         } catch (PDOException $e) {
-            $this->processException($e);
-            return false;
+            throw $this->processException($e);
         }
     }
 
@@ -1661,6 +1660,7 @@ public function deleteDocument(string $collection, string $id): bool
      * @return array<Document>
      * @throws DatabaseException
      * @throws TimeoutException
+     * @throws Exception
      */
     public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER): array
     {
@@ -1804,7 +1804,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         try {
             $stmt->execute();
         } catch (PDOException $e) {
-            $this->processException($e);
+            throw $this->processException($e);
         }
 
         $results = $stmt->fetchAll();
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index a2aea9b30..eabcb5d0d 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -989,10 +989,21 @@ public function testQueryTimeout(): void
         if (!$this->getDatabase()->getAdapter()->getSupportForTimeouts()) {
             $this->expectNotToPerformAssertions();
         }
+
         static::getDatabase()->createCollection('global-timeouts');
-        $this->assertEquals(true, static::getDatabase()->createAttribute('global-timeouts', 'longtext', Database::VAR_STRING, 100000000, true));
 
-        for ($i = 0 ; $i <= 200 ; $i++) {
+        $this->assertEquals(
+            true,
+            static::getDatabase()->createAttribute(
+                collection: 'global-timeouts',
+                id: 'longtext',
+                type: Database::VAR_STRING,
+                size: 100000000,
+                required: true
+            )
+        );
+
+        for ($i = 0 ; $i <= 20 ; $i++) {
             static::getDatabase()->createDocument('global-timeouts', new Document([
                 'longtext' => file_get_contents(__DIR__ . '/../../resources/longtext.txt'),
                 '$permissions' => [

From 9fff3282750bc212291c71799f5e3dcbe850e5b9 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 19:22:04 +1200
Subject: [PATCH 060/100] Fix SQLite tests

---
 src/Database/Adapter/SQLite.php | 98 +++++++++++++++++----------------
 tests/e2e/Adapter/Base.php      | 11 +++-
 2 files changed, 60 insertions(+), 49 deletions(-)

diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index aa971c98e..66b49173b 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -110,12 +110,6 @@ public function createCollection(string $name, array $attributes = [], array $in
     {
         $id = $this->filter($name);
 
-        try {
-            $this->getPDO()->beginTransaction();
-        } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-        }
-
         /** @var array<string> $attributeStrings */
         $attributeStrings = [];
 
@@ -134,8 +128,8 @@ public function createCollection(string $name, array $attributes = [], array $in
 
         $tenantQuery = $this->sharedTables ? '`_tenant` INTEGER DEFAULT NULL,' : '';
 
-        $sql = "
-			CREATE TABLE IF NOT EXISTS `{$this->getSQLTable($id)}` (
+        $collection = "
+			CREATE TABLE {$this->getSQLTable($id)} (
 				`_id` INTEGER PRIMARY KEY AUTOINCREMENT,
 				`_uid` VARCHAR(36) NOT NULL,
 				{$tenantQuery}
@@ -146,32 +140,10 @@ public function createCollection(string $name, array $attributes = [], array $in
 			)
 		";
 
-        $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql);
-
-        $this->getPDO()->prepare($sql)->execute();
+        $collection = $this->trigger(Database::EVENT_COLLECTION_CREATE, $collection);
 
-        $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid'], [], []);
-        $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []);
-        $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []);
-
-        if ($this->sharedTables) {
-            $this->createIndex($id, '_tenant_id', Database::INDEX_KEY, [ '_id'], [], []);
-        }
-
-        foreach ($indexes as $index) {
-            $indexId = $this->filter($index->getId());
-            $indexType = $index->getAttribute('type');
-            $indexAttributes = $index->getAttribute('attributes', []);
-            $indexLengths = $index->getAttribute('lengths', []);
-            $indexOrders = $index->getAttribute('orders', []);
-
-            $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders);
-        }
-
-        $tenantQuery = $this->sharedTables ? '`_tenant` INTEGER DEFAULT NULL,' : '';
-
-        $sql = "
-			CREATE TABLE IF NOT EXISTS `{$this->getSQLTable($id)}_perms` (
+        $permissions = "
+			CREATE TABLE {$this->getSQLTable($id . '_perms')} (
 				`_id` INTEGER PRIMARY KEY AUTOINCREMENT,
 				{$tenantQuery}
 				`_type` VARCHAR(12) NOT NULL,
@@ -180,18 +152,49 @@ public function createCollection(string $name, array $attributes = [], array $in
 			)
 		";
 
-        $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql);
+        $permissions = $this->trigger(Database::EVENT_COLLECTION_CREATE, $permissions);
 
-        $this->getPDO()
-            ->prepare($sql)
-            ->execute();
+        try {
+            $this->getPDO()
+                ->prepare($collection)
+                ->execute();
 
-        $this->createIndex("{$id}_perms", '_index_1', Database::INDEX_UNIQUE, ['_document', '_type', '_permission'], [], []);
-        $this->createIndex("{$id}_perms", '_index_2', Database::INDEX_KEY, ['_permission', '_type'], [], []);
+            $this->getPDO()
+                ->prepare($permissions)
+                ->execute();
 
-        $this->getPDO()->commit();
+            $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid'], [], []);
+            $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []);
+            $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []);
 
-        // Update $this->getCountOfIndexes when adding another default index
+            if ($this->sharedTables) {
+                $this->createIndex($id, '_tenant_id', Database::INDEX_KEY, [ '_id'], [], []);
+            }
+
+            foreach ($indexes as $index) {
+                $indexId = $this->filter($index->getId());
+                $indexType = $index->getAttribute('type');
+                $indexAttributes = $index->getAttribute('attributes', []);
+                $indexLengths = $index->getAttribute('lengths', []);
+                $indexOrders = $index->getAttribute('orders', []);
+
+                $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders);
+            }
+
+            $this->createIndex("{$id}_perms", '_index_1', Database::INDEX_UNIQUE, ['_document', '_type', '_permission'], [], []);
+            $this->createIndex("{$id}_perms", '_index_2', Database::INDEX_KEY, ['_permission', '_type'], [], []);
+
+        } catch (PDOException $e) {
+            $e = $this->processException($e);
+
+            if (!($e instanceof Duplicate)) {
+                $this->getPDO()
+                    ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};")
+                    ->execute();
+            }
+
+            throw $e;
+        }
         return true;
     }
 
@@ -252,14 +255,14 @@ public function deleteCollection(string $id): bool
             $this->getPDO()->rollBack();
         }
 
-        $sql = "DROP TABLE IF EXISTS `{$this->getSQLTable($id)}`";
+        $sql = "DROP TABLE IF EXISTS {$this->getSQLTable($id)}";
         $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql);
 
         $this->getPDO()
             ->prepare($sql)
             ->execute();
 
-        $sql = "DROP TABLE IF EXISTS `{$this->getSQLTable($id)}_perms`";
+        $sql = "DROP TABLE IF EXISTS {$this->getSQLTable($id . '_perms')}";
         $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql);
 
         $this->getPDO()
@@ -327,10 +330,7 @@ public function deleteAttribute(string $collection, string $id, bool $array = fa
             }
         }
 
-        $sql = "
-			ALTER TABLE {$this->getSQLTable($name)}
-			DROP COLUMN `{$id}`
-		";
+        $sql = "ALTER TABLE {$this->getSQLTable($name)} DROP COLUMN `{$id}`";
 
         $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql);
 
@@ -1226,7 +1226,7 @@ protected function getSQLPermissionsCondition(string $collection, array $roles):
      */
     protected function getSQLTable(string $name): string
     {
-        return "{$this->getNamespace()}_{$name}";
+        return "`{$this->getNamespace()}_{$name}`";
     }
 
     /**
@@ -1394,12 +1394,14 @@ protected function processException(PDOException $e): \Exception
          * PDO and Swoole PDOProxy swap error codes and errorInfo
          */
 
+        // Timeout
         if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
             return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         } elseif ($e->getCode() === 3024 && isset($e->errorInfo[0]) && $e->errorInfo[0] === "HY000") {
             return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
+        // Duplicate
         if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         } elseif ($e->getCode() === 1 && isset($e->errorInfo[0]) && $e->errorInfo[0] === 'HY000') {
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index eabcb5d0d..25ffd1798 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -984,10 +984,19 @@ public function testCreatedAtUpdatedAt(): void
     }
 
 
+    /**
+     * @throws AuthorizationException
+     * @throws DuplicateException
+     * @throws ConflictException
+     * @throws LimitException
+     * @throws StructureException
+     * @throws DatabaseException
+     */
     public function testQueryTimeout(): void
     {
         if (!$this->getDatabase()->getAdapter()->getSupportForTimeouts()) {
             $this->expectNotToPerformAssertions();
+            return;
         }
 
         static::getDatabase()->createCollection('global-timeouts');
@@ -1003,7 +1012,7 @@ public function testQueryTimeout(): void
             )
         );
 
-        for ($i = 0 ; $i <= 20 ; $i++) {
+        for ($i = 0; $i < 20; $i++) {
             static::getDatabase()->createDocument('global-timeouts', new Document([
                 'longtext' => file_get_contents(__DIR__ . '/../../resources/longtext.txt'),
                 '$permissions' => [

From fd1bc7f67630db5228b5ad715f0d714c90cb98d6 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 19:24:09 +1200
Subject: [PATCH 061/100] Fix Mirror tests

---
 src/Database/Mirror.php | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index ecbcc329f..4f212fd0c 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -367,7 +367,8 @@ public function updateAttribute(string $collection, string $id, string $type = n
             $array,
             $format,
             $formatOptions,
-            $filters
+            $filters,
+            $newKey,
         );
 
         if ($this->destination === null) {
@@ -396,7 +397,8 @@ public function updateAttribute(string $collection, string $id, string $type = n
                 $document->getAttribute('array'),
                 $document->getAttribute('format'),
                 $document->getAttribute('formatOptions'),
-                $document->getAttribute('filters')
+                $document->getAttribute('filters'),
+                $newKey,
             );
         } catch (\Throwable $err) {
             $this->logError('updateAttribute', $err);

From 8b169a4ceb2de4e4fb75ce50ca04e32f5cdbffaa Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 20:54:39 +1200
Subject: [PATCH 062/100] Fix SQLite tests

---
 tests/e2e/Adapter/Base.php | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 25ffd1798..718d91118 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -62,16 +62,11 @@ public function testPing(): void
 
     public function testCreateExistsDelete(): void
     {
-        $schemaSupport = $this->getDatabase()->getAdapter()->getSupportForSchemas();
-        if (!$schemaSupport) {
-            $this->assertEquals(static::getDatabase(), static::getDatabase()->setDatabase($this->testDatabase));
-            $this->assertEquals(true, static::getDatabase()->create());
+        if (!static::getDatabase()->getAdapter()->getSupportForSchemas()) {
+            $this->expectNotToPerformAssertions();
             return;
         }
 
-        if (!static::getDatabase()->exists($this->testDatabase)) {
-            $this->assertEquals(true, static::getDatabase()->create());
-        }
         $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase));
         $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase));
         $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase));

From 088d549407657e690877386d34e9d813d68f3472 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 21:03:03 +1200
Subject: [PATCH 063/100] Fix missing silent calls in mirror

---
 src/Database/Mirror.php    | 29 ++++++++++++++++-------------
 tests/e2e/Adapter/Base.php |  2 +-
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 4f212fd0c..aec8beb0c 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -226,13 +226,15 @@ public function createCollection(string $id, array $attributes = [], array $inde
                 $documentSecurity
             );
 
-            $this->createUpgrades();
-
-            $this->source->createDocument('upgrades', new Document([
-                '$id' => $id,
-                'collectionId' => $id,
-                'status' => 'upgraded'
-            ]));
+            $this->silent(function () use ($id) {
+                $this->createUpgrades();
+
+                $this->source->createDocument('upgrades', new Document([
+                    '$id' => $id,
+                    'collectionId' => $id,
+                    'status' => 'upgraded'
+                ]));
+            });
         } catch (\Throwable $err) {
             $this->logError('createCollection', $err);
         }
@@ -512,7 +514,7 @@ public function createDocument(string $collection, Document $document): Document
             return $document;
         }
 
-        $upgrade = $this->getUpgradeStatus($collection);
+        $upgrade = $this->silent(fn() => $this->getUpgradeStatus($collection));
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $document;
         }
@@ -562,7 +564,7 @@ public function createDocuments(
             return $documents;
         }
 
-        $upgrade = $this->getUpgradeStatus($collection);
+        $upgrade = $this->silent(fn () => $this->getUpgradeStatus($collection));
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $documents;
         }
@@ -618,7 +620,8 @@ public function updateDocument(string $collection, string $id, Document $documen
             return $document;
         }
 
-        $upgrade = $this->getUpgradeStatus($collection);
+        $upgrade = $this->silent(fn() => $this->getUpgradeStatus($collection));
+
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $document;
         }
@@ -668,7 +671,7 @@ public function updateDocuments(
             return $documents;
         }
 
-        $upgrade = $this->getUpgradeStatus($collection);
+        $upgrade = $this->silent(fn() => $this->getUpgradeStatus($collection));
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $documents;
         }
@@ -723,7 +726,7 @@ public function deleteDocument(string $collection, string $id): bool
             return $result;
         }
 
-        $upgrade = $this->getUpgradeStatus($collection);
+        $upgrade = $this->silent(fn () => $this->getUpgradeStatus($collection));
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $result;
         }
@@ -900,7 +903,7 @@ protected function getUpgradeStatus(string $collection): ?Document
             try {
                 return $this->getDocument('upgrades', $collection);
             } catch (\Throwable) {
-                return;
+                return null;
             }
         });
     }
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 718d91118..9d97ec0bf 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -15392,7 +15392,7 @@ public function testEvents(): void
                 $database->setDatabase('hellodb');
                 $database->create();
             } else {
-                array_shift($events);
+                \array_shift($events);
             }
 
             $database->list();

From bad0ea537637fff0546287c92e4a3242cd129b8d Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 19 Aug 2024 21:05:54 +1200
Subject: [PATCH 064/100] Lint

---
 src/Database/Mirror.php | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index aec8beb0c..6ff3860a7 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -514,7 +514,7 @@ public function createDocument(string $collection, Document $document): Document
             return $document;
         }
 
-        $upgrade = $this->silent(fn() => $this->getUpgradeStatus($collection));
+        $upgrade = $this->silent(fn () => $this->getUpgradeStatus($collection));
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $document;
         }
@@ -620,7 +620,7 @@ public function updateDocument(string $collection, string $id, Document $documen
             return $document;
         }
 
-        $upgrade = $this->silent(fn() => $this->getUpgradeStatus($collection));
+        $upgrade = $this->silent(fn () => $this->getUpgradeStatus($collection));
 
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $document;
@@ -671,7 +671,7 @@ public function updateDocuments(
             return $documents;
         }
 
-        $upgrade = $this->silent(fn() => $this->getUpgradeStatus($collection));
+        $upgrade = $this->silent(fn () => $this->getUpgradeStatus($collection));
         if ($upgrade === null || $upgrade->getAttribute('status', '') !== 'upgraded') {
             return $documents;
         }
@@ -903,7 +903,7 @@ protected function getUpgradeStatus(string $collection): ?Document
             try {
                 return $this->getDocument('upgrades', $collection);
             } catch (\Throwable) {
-                return null;
+                return;
             }
         });
     }

From 67bf1fef4dff6503b897a905bdb5fbd200587e6b Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 21 Aug 2024 20:11:14 +1200
Subject: [PATCH 065/100] Merge pull request #439 from
 utopia-php/feat-atomic-updates

Feat atomic updates
# Conflicts:
#	composer.json
#	composer.lock
#	src/Database/Adapter/MariaDB.php
#	src/Database/Adapter/Postgres.php
#	src/Database/Adapter/SQLite.php
#	src/Database/Database.php
#	tests/e2e/Adapter/Base.php
---
 composer.json                     |  16 +-
 composer.lock                     |  46 ++--
 src/Database/Adapter.php          |  71 ++++-
 src/Database/Adapter/MariaDB.php  |  50 +---
 src/Database/Adapter/Mongo.php    |  22 +-
 src/Database/Adapter/Postgres.php | 104 ++------
 src/Database/Adapter/SQL.php      |  98 ++++++-
 src/Database/Adapter/SQLite.php   |  49 +---
 src/Database/Database.php         | 421 ++++++++++++++++++------------
 9 files changed, 503 insertions(+), 374 deletions(-)

diff --git a/composer.json b/composer.json
index 52f8a44f8..d0097c5d5 100755
--- a/composer.json
+++ b/composer.json
@@ -41,14 +41,14 @@
         "utopia-php/mongo": "0.3.*"
     },
     "require-dev": {
-        "fakerphp/faker": "^1.14",
-        "phpunit/phpunit": "^9.4",
-        "pcov/clobber": "^2.0",
-        "swoole/ide-helper": "5.1.2",
-        "utopia-php/cli": "^0.14.0",
-        "laravel/pint": "1.13.*",
-        "phpstan/phpstan": "1.10.*",
-        "rregeer/phpunit-coverage-check": "^0.3.1"
+        "fakerphp/faker": "1.23.*",
+        "phpunit/phpunit": "9.6.*",
+        "pcov/clobber": "2.0.*",
+        "swoole/ide-helper": "5.1.3",
+        "utopia-php/cli": "0.14.*",
+        "laravel/pint": "1.17.*",
+        "phpstan/phpstan": "1.11.*",
+        "rregeer/phpunit-coverage-check": "0.3.*"
     },
     "suggests": {
         "ext-mongodb": "Needed to support MongoDB Database Adapter",
diff --git a/composer.lock b/composer.lock
index 51c48d890..5bd08c299 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "1ef7c11d9a0628b3c57c8c80dbebbba1",
+    "content-hash": "8a6537ec5c4f47cd0b6c34e9ceda042b",
     "packages": [
         {
             "name": "jean85/pretty-package-versions",
@@ -506,16 +506,16 @@
         },
         {
             "name": "laravel/pint",
-            "version": "v1.13.11",
+            "version": "v1.17.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/laravel/pint.git",
-                "reference": "60a163c3e7e3346a1dec96d3e6f02e6465452552"
+                "reference": "e8a88130a25e3f9d4d5785e6a1afca98268ab110"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/laravel/pint/zipball/60a163c3e7e3346a1dec96d3e6f02e6465452552",
-                "reference": "60a163c3e7e3346a1dec96d3e6f02e6465452552",
+                "url": "https://api.github.com/repos/laravel/pint/zipball/e8a88130a25e3f9d4d5785e6a1afca98268ab110",
+                "reference": "e8a88130a25e3f9d4d5785e6a1afca98268ab110",
                 "shasum": ""
             },
             "require": {
@@ -526,13 +526,13 @@
                 "php": "^8.1.0"
             },
             "require-dev": {
-                "friendsofphp/php-cs-fixer": "^3.49.0",
-                "illuminate/view": "^10.43.0",
-                "larastan/larastan": "^2.8.1",
-                "laravel-zero/framework": "^10.3.0",
-                "mockery/mockery": "^1.6.7",
+                "friendsofphp/php-cs-fixer": "^3.61.1",
+                "illuminate/view": "^10.48.18",
+                "larastan/larastan": "^2.9.8",
+                "laravel-zero/framework": "^10.4.0",
+                "mockery/mockery": "^1.6.12",
                 "nunomaduro/termwind": "^1.15.1",
-                "pestphp/pest": "^2.33.6"
+                "pestphp/pest": "^2.35.0"
             },
             "bin": [
                 "builds/pint"
@@ -568,7 +568,7 @@
                 "issues": "https://github.com/laravel/pint/issues",
                 "source": "https://github.com/laravel/pint"
             },
-            "time": "2024-02-13T17:20:13+00:00"
+            "time": "2024-08-06T15:11:54+00:00"
         },
         {
             "name": "myclabs/deep-copy",
@@ -840,16 +840,16 @@
         },
         {
             "name": "phpstan/phpstan",
-            "version": "1.10.67",
+            "version": "1.11.11",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpstan/phpstan.git",
-                "reference": "16ddbe776f10da6a95ebd25de7c1dbed397dc493"
+                "reference": "707c2aed5d8d0075666e673a5e71440c1d01a5a3"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpstan/phpstan/zipball/16ddbe776f10da6a95ebd25de7c1dbed397dc493",
-                "reference": "16ddbe776f10da6a95ebd25de7c1dbed397dc493",
+                "url": "https://api.github.com/repos/phpstan/phpstan/zipball/707c2aed5d8d0075666e673a5e71440c1d01a5a3",
+                "reference": "707c2aed5d8d0075666e673a5e71440c1d01a5a3",
                 "shasum": ""
             },
             "require": {
@@ -894,7 +894,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2024-04-16T07:22:02+00:00"
+            "time": "2024-08-19T14:37:29+00:00"
         },
         {
             "name": "phpunit/php-code-coverage",
@@ -2382,16 +2382,16 @@
         },
         {
             "name": "swoole/ide-helper",
-            "version": "5.1.2",
+            "version": "5.1.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/swoole/ide-helper.git",
-                "reference": "33ec7af9111b76d06a70dd31191cc74793551112"
+                "reference": "9cfc6669b83be0fa6fface91a6f372a0bb84bf1a"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/swoole/ide-helper/zipball/33ec7af9111b76d06a70dd31191cc74793551112",
-                "reference": "33ec7af9111b76d06a70dd31191cc74793551112",
+                "url": "https://api.github.com/repos/swoole/ide-helper/zipball/9cfc6669b83be0fa6fface91a6f372a0bb84bf1a",
+                "reference": "9cfc6669b83be0fa6fface91a6f372a0bb84bf1a",
                 "shasum": ""
             },
             "type": "library",
@@ -2408,9 +2408,9 @@
             "description": "IDE help files for Swoole.",
             "support": {
                 "issues": "https://github.com/swoole/ide-helper/issues",
-                "source": "https://github.com/swoole/ide-helper/tree/5.1.2"
+                "source": "https://github.com/swoole/ide-helper/tree/5.1.3"
             },
-            "time": "2024-02-01T22:28:11+00:00"
+            "time": "2024-06-17T05:45:20+00:00"
         },
         {
             "name": "symfony/deprecation-contracts",
diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php
index 00d8436bf..05d651c8d 100644
--- a/src/Database/Adapter.php
+++ b/src/Database/Adapter.php
@@ -17,6 +17,8 @@ abstract class Adapter
 
     protected ?int $tenant = null;
 
+    protected int $inTransaction = 0;
+
     /**
      * @var array<string, mixed>
      */
@@ -232,6 +234,70 @@ public function resetMetadata(): static
         return $this;
     }
 
+    /**
+     * Start a new transaction.
+     *
+     * If a transaction is already active, this will only increment the transaction count and return true.
+     *
+     * @return bool
+     * @throws DatabaseException
+     */
+    abstract public function startTransaction(): bool;
+
+    /**
+     * Commit a transaction.
+     *
+     * If no transaction is active, this will be a no-op and will return false.
+     * If there is more than one active transaction, this decrement the transaction count and return true.
+     * If the transaction count is 1, it will be commited, the transaction count will be reset to 0, and return true.
+     *
+     * @return bool
+     * @throws DatabaseException
+     */
+    abstract public function commitTransaction(): bool;
+
+    /**
+     * Rollback a transaction.
+     *
+     * If no transaction is active, this will be a no-op and will return false.
+     * If 1 or more transactions are active, this will roll back all transactions, reset the count to 0, and return true.
+     *
+     * @return bool
+     * @throws DatabaseException
+     */
+    abstract public function rollbackTransaction(): bool;
+
+    /**
+     * Check if a transaction is active.
+     *
+     * @return bool
+     * @throws DatabaseException
+     */
+    public function inTransaction(): bool
+    {
+        return $this->inTransaction > 0;
+    }
+
+    /**
+     * @template T
+     * @param callable(): T $callback
+     * @return T
+     * @throws \Throwable
+     */
+    public function withTransaction(callable $callback): mixed
+    {
+        $this->startTransaction();
+
+        try {
+            $result = $callback();
+            $this->commitTransaction();
+            return $result;
+        } catch (\Throwable $e) {
+            $this->rollbackTransaction();
+            throw $e;
+        }
+    }
+
     /**
      * Apply a transformation to a query before an event occurs
      *
@@ -460,9 +526,10 @@ abstract public function deleteIndex(string $collection, string $id): bool;
      * @param string $collection
      * @param string $id
      * @param array<Query> $queries
+     * @param bool $forUpdate
      * @return Document
      */
-    abstract public function getDocument(string $collection, string $id, array $queries = []): Document;
+    abstract public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document;
 
     /**
      * Create Document
@@ -669,6 +736,8 @@ abstract public function getSupportForTimeouts(): bool;
      */
     abstract public function getSupportForRelationships(): bool;
 
+    abstract public function getSupportForUpdateLock(): bool;
+
     /**
      * Get current attribute count from collection document
      *
diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index e85cd236c..b7ba2ff6c 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -772,8 +772,6 @@ public function deleteIndex(string $collection, string $id): bool
     public function createDocument(string $collection, Document $document): Document
     {
         try {
-            $this->getPDO()->beginTransaction();
-
             $attributes = $document->getAttributes();
             $attributes['_createdAt'] = $document->getCreatedAt();
             $attributes['_updatedAt'] = $document->getUpdatedAt();
@@ -879,15 +877,7 @@ public function createDocument(string $collection, Document $document): Document
             if (isset($stmtPermissions)) {
                 $stmtPermissions->execute();
             }
-
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
         } catch (\Throwable $e) {
-            if($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
-
             if($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
@@ -921,7 +911,6 @@ public function createDocuments(string $collection, array $documents, int $batch
         }
 
         try {
-            $this->getPDO()->beginTransaction();
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, \max(1, $batchSize));
             $internalIds = [];
@@ -1019,15 +1008,7 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $stmtPermissions?->execute();
                 }
             }
-
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
         } catch (\Throwable $e) {
-            if($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
-
             if($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
@@ -1066,8 +1047,6 @@ public function createDocuments(string $collection, array $documents, int $batch
     public function updateDocument(string $collection, Document $document): Document
     {
         try {
-            $this->getPDO()->beginTransaction();
-
             $attributes = $document->getAttributes();
             $attributes['_createdAt'] = $document->getCreatedAt();
             $attributes['_updatedAt'] = $document->getUpdatedAt();
@@ -1090,6 +1069,8 @@ public function updateDocument(string $collection, Document $document): Document
                 $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
             }
 
+            $sql .= ' FOR UPDATE';
+
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
 
             /**
@@ -1285,14 +1266,7 @@ public function updateDocument(string $collection, Document $document): Document
                 $stmtAddPermissions->execute();
             }
 
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
         } catch (\Throwable $e) {
-            if($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
-
             if($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
@@ -1326,8 +1300,6 @@ public function updateDocuments(string $collection, array $documents, int $batch
         }
 
         try {
-            $this->getPDO()->beginTransaction();
-
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, max(1, $batchSize));
 
@@ -1551,15 +1523,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     $stmtAddPermissions->execute();
                 }
             }
-
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
         } catch (\Throwable $e) {
-            if($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
-
             if($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
@@ -1636,8 +1600,6 @@ public function increaseDocumentAttribute(string $collection, string $id, string
     public function deleteDocument(string $collection, string $id): bool
     {
         try {
-            $this->getPDO()->beginTransaction();
-
             $name = $this->filter($collection);
 
             $sql = "
@@ -1686,15 +1648,7 @@ public function deleteDocument(string $collection, string $id): bool
             if (!$stmtPermissions->execute()) {
                 throw new DatabaseException('Failed to delete permissions');
             }
-
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
         } catch (\Throwable $e) {
-            if($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
-
             throw new DatabaseException($e->getMessage(), $e->getCode(), $e);
         }
 
diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
index ba84b942a..9e48c9a41 100644
--- a/src/Database/Adapter/Mongo.php
+++ b/src/Database/Adapter/Mongo.php
@@ -58,6 +58,21 @@ public function __construct(Client $client)
         $this->client->connect();
     }
 
+    public function startTransaction(): bool
+    {
+        return true;
+    }
+
+    public function commitTransaction(): bool
+    {
+        return true;
+    }
+
+    public function rollbackTransaction(): bool
+    {
+        return true;
+    }
+
     /**
      * Ping Database
      *
@@ -625,7 +640,7 @@ public function deleteIndex(string $collection, string $id): bool
      * @return Document
      * @throws MongoException
      */
-    public function getDocument(string $collection, string $id, array $queries = []): Document
+    public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document
     {
         $name = $this->getNamespace() . '_' . $this->filter($collection);
 
@@ -1641,6 +1656,11 @@ public function getSupportForRelationships(): bool
         return false;
     }
 
+    public function getSupportForUpdateLock(): bool
+    {
+        return false;
+    }
+
     /**
      * Get current attribute count from collection document
      *
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index bb75b23c5..edbb00203 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -374,61 +374,47 @@ public function renameAttribute(string $collection, string $old, string $new): b
      */
     public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false, string $newKey = null): bool
     {
-        try {
-            $name = $this->filter($collection);
-            $id = $this->filter($id);
-            $type = $this->getSQLType($type, $size, $signed, $array);
-
-            $this->getPDO()->beginTransaction();
+        $name = $this->filter($collection);
+        $id = $this->filter($id);
+        $type = $this->getSQLType($type, $size, $signed, $array);
 
-            if ($type == 'TIMESTAMP(3)') {
-                $type = "TIMESTAMP(3) without time zone USING TO_TIMESTAMP(\"$id\", 'YYYY-MM-DD HH24:MI:SS.MS')";
-            }
+        if ($type == 'TIMESTAMP(3)') {
+            $type = "TIMESTAMP(3) without time zone USING TO_TIMESTAMP(\"$id\", 'YYYY-MM-DD HH24:MI:SS.MS')";
+        }
 
-            if (!empty($newKey) && $id !== $newKey) {
-                $newKey = $this->filter($newKey);
+        if (!empty($newKey) && $id !== $newKey) {
+            $newKey = $this->filter($newKey);
 
-                $sql = "
+            $sql = "
                     ALTER TABLE {$this->getSQLTable($name)}
                     RENAME COLUMN \"{$id}\" TO \"{$newKey}\"
                 ";
 
-                $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql);
-
-                $result = $this->getPDO()
-                    ->prepare($sql)
-                    ->execute();
+            $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql);
 
-                if (!$result) {
-                    return false;
-                }
+            $result = $this->getPDO()
+                ->prepare($sql)
+                ->execute();
 
-                $id = $newKey;
+            if (!$result) {
+                return false;
             }
 
-            $sql = "
+            $id = $newKey;
+        }
+
+        $sql = "
                 ALTER TABLE {$this->getSQLTable($name)}
                 ALTER COLUMN \"{$id}\" TYPE {$type}
             ";
 
-            $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql);
-
-            $result = $this->getPDO()
-                ->prepare($sql)
-                ->execute();
-
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
+        $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql);
 
-            return $result;
-        } catch (\Throwable $e) {
-            if($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
+        $result = $this->getPDO()
+            ->prepare($sql)
+            ->execute();
 
-            throw $e;
-        }
+        return $result;
     }
 
     /**
@@ -801,8 +787,6 @@ public function createDocument(string $collection, Document $document): Document
         $columns = '';
         $columnNames = '';
 
-        $this->getPDO()->beginTransaction();
-
         /**
          * Insert Attributes
          */
@@ -888,17 +872,12 @@ public function createDocument(string $collection, Document $document): Document
         } catch (Throwable $e) {
             switch ($e->getCode()) {
                 case 23505:
-                    $this->getPDO()->rollBack();
-                    throw new DuplicateException('Duplicated document: ' . $e->getMessage());
+                    throw new Duplicate('Duplicated document: ' . $e->getMessage());
                 default:
                     throw $e;
             }
         }
 
-        if (!$this->getPDO()->commit()) {
-            throw new DatabaseException('Failed to commit transaction');
-        }
-
         return $document;
     }
 
@@ -919,9 +898,7 @@ public function createDocuments(string $collection, array $documents, int $batch
             return $documents;
         }
 
-
         try {
-            $this->getPDO()->beginTransaction();
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, max(1, $batchSize));
             $internalIds = [];
@@ -1000,12 +977,8 @@ public function createDocuments(string $collection, array $documents, int $batch
                 }
             }
 
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
+            return $documents;
         } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-
             throw match ($e->getCode()) {
                 1062, 23000 => new DuplicateException('Duplicated document: ' . $e->getMessage()),
                 default => $e,
@@ -1059,6 +1032,8 @@ public function updateDocument(string $collection, Document $document): Document
             $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
+        $sql .= ' FOR UPDATE';
+
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
 
         /**
@@ -1086,8 +1061,6 @@ public function updateDocument(string $collection, Document $document): Document
             return $carry;
         }, $initial);
 
-        $this->getPDO()->beginTransaction();
-
         /**
          * Get removed Permissions
          */
@@ -1242,9 +1215,6 @@ public function updateDocument(string $collection, Document $document): Document
                 $stmtAddPermissions->execute();
             }
         } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-
-            // Must be a switch for loose match
             switch ($e->getCode()) {
                 case 1062:
                 case 23505:
@@ -1254,10 +1224,6 @@ public function updateDocument(string $collection, Document $document): Document
             }
         }
 
-        if (!$this->getPDO()->commit()) {
-            throw new DatabaseException('Failed to commit transaction');
-        }
-
         return $document;
     }
 
@@ -1277,8 +1243,6 @@ public function updateDocuments(string $collection, array $documents, int $batch
             return $documents;
         }
 
-        $this->getPDO()->beginTransaction();
-
         try {
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, max(1, $batchSize));
@@ -1501,13 +1465,8 @@ public function updateDocuments(string $collection, array $documents, int $batch
                 }
             }
 
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
-
             return $documents;
         } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
 
             // Must be a switch for loose match
             switch ($e->getCode()) {
@@ -1582,8 +1541,6 @@ public function deleteDocument(string $collection, string $id): bool
     {
         $name = $this->filter($collection);
 
-        $this->getPDO()->beginTransaction();
-
         $sql = "
 			DELETE FROM {$this->getSQLTable($name)} 
 			WHERE _uid = :_uid
@@ -1632,14 +1589,9 @@ public function deleteDocument(string $collection, string $id): bool
                 throw new DatabaseException('Failed to delete permissions');
             }
         } catch (\Throwable $th) {
-            $this->getPDO()->rollBack();
             throw new DatabaseException($th->getMessage());
         }
 
-        if (!$this->getPDO()->commit()) {
-            throw new DatabaseException('Failed to commit transaction');
-        }
-
         return $deleted;
     }
 
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index de639036d..e115afdb3 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -27,6 +27,81 @@ public function __construct(mixed $pdo)
         $this->pdo = $pdo;
     }
 
+    /**
+     * @inheritDoc
+     */
+    public function startTransaction(): bool
+    {
+        try {
+            if ($this->inTransaction === 0) {
+                $result = $this->getPDO()->beginTransaction();
+            } else {
+                $result = true;
+            }
+        } catch (PDOException $e) {
+            throw new DatabaseException('Failed to start transaction: ' . $e->getMessage(), $e->getCode(), $e);
+        }
+
+        if (!$result) {
+            throw new DatabaseException('Failed to start transaction');
+        }
+
+        $this->inTransaction++;
+
+        return $result;
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function commitTransaction(): bool
+    {
+        if ($this->inTransaction === 0) {
+            return false;
+        } elseif ($this->inTransaction > 1) {
+            $this->inTransaction--;
+            return true;
+        }
+
+        try {
+            $result = $this->getPDO()->commit();
+        } catch (PDOException $e) {
+            throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e);
+        } finally {
+            $this->inTransaction--;
+        }
+
+        if (!$result) {
+            throw new DatabaseException('Failed to commit transaction');
+        }
+
+        return $result;
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function rollbackTransaction(): bool
+    {
+        if ($this->inTransaction === 0) {
+            return false;
+        }
+
+        try {
+            $result = $this->getPDO()->rollBack();
+        } catch (PDOException $e) {
+            throw new DatabaseException('Failed to rollback transaction: ' . $e->getMessage(), $e->getCode(), $e);
+        } finally {
+            $this->inTransaction = 0;
+        }
+
+        if (!$result) {
+            throw new DatabaseException('Failed to rollback transaction');
+        }
+
+        return $result;
+    }
+
     /**
      * Ping Database
      *
@@ -101,16 +176,19 @@ public function list(): array
      * @param string $collection
      * @param string $id
      * @param Query[] $queries
+     * @param bool $forUpdate
      * @return Document
-     * @throws Exception
+     * @throws DatabaseException
      */
-    public function getDocument(string $collection, string $id, array $queries = []): Document
+    public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document
     {
         $name = $this->filter($collection);
         $selections = $this->getAttributeSelections($queries);
 
+        $forUpdate = $forUpdate ? 'FOR UPDATE' : '';
+
         $sql = "
-		    SELECT {$this->getAttributeProjection($selections)} 
+		    SELECT {$this->getAttributeProjection($selections)}
             FROM {$this->getSQLTable($name)}
             WHERE _uid = :_uid 
 		";
@@ -119,6 +197,10 @@ public function getDocument(string $collection, string $id, array $queries = [])
             $sql .= "AND (_tenant = :_tenant OR _tenant IS NULL)";
         }
 
+        if ($this->getSupportForUpdateLock()) {
+            $sql .= " {$forUpdate}";
+        }
+
         $stmt = $this->getPDO()->prepare($sql);
 
         $stmt->bindValue(':_uid', $id);
@@ -259,6 +341,16 @@ public function getSupportForFulltextIndex(): bool
         return true;
     }
 
+    /**
+     * Are FOR UPDATE locks supported?
+     *
+     * @return bool
+     */
+    public function getSupportForUpdateLock(): bool
+    {
+        return true;
+    }
+
     /**
      * Get current attribute count from collection document
      *
diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index 66b49173b..e9f4de36e 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -167,6 +167,9 @@ public function createCollection(string $name, array $attributes = [], array $in
             $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []);
             $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []);
 
+            $this->createIndex("{$id}_perms", '_index_1', Database::INDEX_UNIQUE, ['_document', '_type', '_permission'], [], []);
+            $this->createIndex("{$id}_perms", '_index_2', Database::INDEX_KEY, ['_permission', '_type'], [], []);
+
             if ($this->sharedTables) {
                 $this->createIndex($id, '_tenant_id', Database::INDEX_KEY, [ '_id'], [], []);
             }
@@ -249,13 +252,7 @@ public function deleteCollection(string $id): bool
     {
         $id = $this->filter($id);
 
-        try {
-            $this->getPDO()->beginTransaction();
-        } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-        }
-
-        $sql = "DROP TABLE IF EXISTS {$this->getSQLTable($id)}";
+        $sql = "DROP TABLE IF EXISTS `{$this->getSQLTable($id)}`";
         $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql);
 
         $this->getPDO()
@@ -269,8 +266,6 @@ public function deleteCollection(string $id): bool
             ->prepare($sql)
             ->execute();
 
-        $this->getPDO()->commit();
-
         return true;
     }
 
@@ -472,12 +467,6 @@ public function createDocument(string $collection, Document $document): Document
         $columns = ['_uid'];
         $values = ['_uid'];
 
-        try {
-            $this->getPDO()->beginTransaction();
-        } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-        }
-
         /**
          * Insert Attributes
          */
@@ -562,16 +551,12 @@ public function createDocument(string $collection, Document $document): Document
                 $stmtPermissions->execute();
             }
         } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
             throw match ($e->getCode()) {
                 "1062", "23000" => new Duplicate('Duplicated document: ' . $e->getMessage()),
                 default => $e,
             };
         }
 
-        if (!$this->getPDO()->commit()) {
-            throw new DatabaseException('Failed to commit transaction');
-        }
 
         return $document;
     }
@@ -600,7 +585,6 @@ public function updateDocument(string $collection, Document $document): Document
         $name = $this->filter($collection);
         $columns = '';
 
-
         $sql = "
 			SELECT _type, _permission
 			FROM `{$this->getNamespace()}_{$name}_perms`
@@ -638,12 +622,6 @@ public function updateDocument(string $collection, Document $document): Document
             return $carry;
         }, $initial);
 
-        try {
-            $this->getPDO()->beginTransaction();
-        } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-        }
-
         /**
          * Get removed Permissions
          */
@@ -798,8 +776,6 @@ public function updateDocument(string $collection, Document $document): Document
                 $stmtAddPermissions->execute();
             }
         } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-
             throw match ($e->getCode()) {
                 '1062',
                 '23000' => new Duplicate('Duplicated document: ' . $e->getMessage()),
@@ -807,10 +783,6 @@ public function updateDocument(string $collection, Document $document): Document
             };
         }
 
-        if (!$this->getPDO()->commit()) {
-            throw new DatabaseException('Failed to commit transaction');
-        }
-
         return $document;
     }
 
@@ -831,8 +803,6 @@ public function updateDocuments(string $collection, array $documents, int $batch
             return $documents;
         }
 
-        $this->getPDO()->beginTransaction();
-
         try {
             $name = $this->filter($collection);
             $batches = \array_chunk($documents, max(1, $batchSize));
@@ -1061,14 +1031,8 @@ public function updateDocuments(string $collection, array $documents, int $batch
                 }
             }
 
-            if (!$this->getPDO()->commit()) {
-                throw new DatabaseException('Failed to commit transaction');
-            }
-
             return $documents;
         } catch (PDOException $e) {
-            $this->getPDO()->rollBack();
-
             throw match ($e->getCode()) {
                 1062, 23000 => new Duplicate('Duplicated document: ' . $e->getMessage()),
                 default => $e,
@@ -1126,6 +1090,11 @@ public function getSupportForRelationships(): bool
         return false;
     }
 
+    public function getSupportForUpdateLock(): bool
+    {
+        return false;
+    }
+
     /**
      * Get SQL Index Type
      *
diff --git a/src/Database/Database.php b/src/Database/Database.php
index fbf92c4eb..61b2f2b98 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -935,6 +935,59 @@ public function getAdapter(): Adapter
         return $this->adapter;
     }
 
+    /**
+     * Start a new transaction.
+     *
+     * If a transaction is already active, this will only increment the transaction count and return true.
+     *
+     * @return bool
+     * @throws DatabaseException
+     */
+    public function startTransaction(): bool
+    {
+        return $this->adapter->startTransaction();
+    }
+
+    /**
+     * Commit a transaction.
+     *
+     * If no transaction is active, this will be a no-op and will return false.
+     * If there is more than one active transaction, this decrement the transaction count and return true.
+     * If the transaction count is 1, it will be commited, the transaction count will be reset to 0, and return true.
+     *
+     * @return bool
+     * @throws DatabaseException
+     */
+    public function commitTransaction(): bool
+    {
+        return $this->adapter->startTransaction();
+    }
+
+    /**
+     * Rollback a transaction.
+     *
+     * If no transaction is active, this will be a no-op and will return false.
+     * If 1 or more transactions are active, this will roll back all transactions, reset the count to 0, and return true.
+     *
+     * @return bool
+     * @throws DatabaseException
+     */
+    public function rollbackTransaction(): bool
+    {
+        return $this->adapter->rollbackTransaction();
+    }
+
+    /**
+     * @template T
+     * @param callable(): T $callback
+     * @return T
+     * @throws \Throwable
+     */
+    public function withTransaction(callable $callback): mixed
+    {
+        return $this->adapter->withTransaction($callback);
+    }
+
     /**
      * Ping Database
      *
@@ -2659,7 +2712,7 @@ public function deleteIndex(string $collection, string $id): bool
      * @throws DatabaseException
      * @throws Exception
      */
-    public function getDocument(string $collection, string $id, array $queries = []): Document
+    public function getDocument(string $collection, string $id, array $queries = [], bool $forUpdate = false): Document
     {
         if ($collection === self::METADATA && $id === self::METADATA) {
             return new Document(self::COLLECTION);
@@ -2763,7 +2816,7 @@ public function getDocument(string $collection, string $id, array $queries = [])
             return $document;
         }
 
-        $document = $this->adapter->getDocument($collection->getId(), $id, $queries);
+        $document = $this->adapter->getDocument($collection->getId(), $id, $queries, $forUpdate);
 
         if ($document->isEmpty()) {
             return $document;
@@ -3138,11 +3191,13 @@ public function createDocument(string $collection, Document $document): Document
             throw new StructureException($structure->getDescription());
         }
 
-        if ($this->resolveRelationships) {
-            $document = $this->silent(fn () => $this->createDocumentRelationships($collection, $document));
-        }
+        $document = $this->withTransaction(function () use ($collection, $document) {
+            if ($this->resolveRelationships) {
+                $document = $this->silent(fn () => $this->createDocumentRelationships($collection, $document));
+            }
 
-        $document = $this->adapter->createDocument($collection->getId(), $document);
+            return $this->adapter->createDocument($collection->getId(), $document);
+        });
 
         if ($this->resolveRelationships) {
             $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document));
@@ -3202,7 +3257,9 @@ public function createDocuments(string $collection, array $documents, int $batch
             $documents[$key] = $document;
         }
 
-        $documents = $this->adapter->createDocuments($collection->getId(), $documents, $batchSize);
+        $documents = $this->withTransaction(function () use ($collection, $documents, $batchSize) {
+            return $this->adapter->createDocuments($collection->getId(), $documents, $batchSize);
+        });
 
         foreach ($documents as $key => $document) {
             if ($this->resolveRelationships) {
@@ -3559,165 +3616,172 @@ public function updateDocument(string $collection, string $id, Document $documen
             throw new DatabaseException('Must define $id attribute');
         }
 
-        $time = DateTime::now();
-        $old = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); // Skip ensures user does not need read permission for this
+        $collection = $this->silent(fn () => $this->getCollection($collection));
 
-        $document = \array_merge($old->getArrayCopy(), $document->getArrayCopy());
-        $document['$createdAt'] = $old->getCreatedAt();                 // Make sure user doesn't switch createdAt
-        $document['$collection'] = $old->getAttribute('$collection');   // Make sure user doesn't switch collection ID
+        $document = $this->withTransaction(function () use ($collection, $id, $document) {
+            $time = DateTime::now();
+            $old = Authorization::skip(fn () => $this->silent(
+                fn () =>
+                $this->getDocument($collection->getId(), $id, forUpdate: true)
+            ));
 
-        if($this->adapter->getSharedTables()) {
-            $document['$tenant'] = $old->getAttribute('$tenant');       // Make sure user doesn't switch tenant
-        }
+            $document = \array_merge($old->getArrayCopy(), $document->getArrayCopy());
+            $document['$collection'] = $old->getAttribute('$collection');   // Make sure user doesn't switch collection ID
+            $document['$createdAt'] = $old->getCreatedAt();                 // Make sure user doesn't switch createdAt
 
-        $document = new Document($document);
+            if ($this->adapter->getSharedTables()) {
+                $document['$tenant'] = $old->getAttribute('$tenant');           // Make sure user doesn't switch tenant
+            }
 
-        $collection = $this->silent(fn () => $this->getCollection($collection));
+            $document = new Document($document);
 
-        $relationships = \array_filter($collection->getAttribute('attributes', []), function ($attribute) {
-            return $attribute['type'] === Database::VAR_RELATIONSHIP;
-        });
+            $relationships = \array_filter($collection->getAttribute('attributes', []), function ($attribute) {
+                return $attribute['type'] === Database::VAR_RELATIONSHIP;
+            });
 
-        $updateValidator = new Authorization(self::PERMISSION_UPDATE);
-        $readValidator = new Authorization(self::PERMISSION_READ);
-        $shouldUpdate = false;
+            $updateValidator = new Authorization(self::PERMISSION_UPDATE);
+            $readValidator = new Authorization(self::PERMISSION_READ);
+            $shouldUpdate = false;
 
-        if ($collection->getId() !== self::METADATA) {
-            $documentSecurity = $collection->getAttribute('documentSecurity', false);
+            if ($collection->getId() !== self::METADATA) {
+                $documentSecurity = $collection->getAttribute('documentSecurity', false);
 
-            foreach ($relationships as $relationship) {
-                $relationships[$relationship->getAttribute('key')] = $relationship;
-            }
+                foreach ($relationships as $relationship) {
+                    $relationships[$relationship->getAttribute('key')] = $relationship;
+                }
 
-            // Compare if the document has any changes
-            foreach ($document as $key => $value) {
-                // Skip the nested documents as they will be checked later in recursions.
-                if (\array_key_exists($key, $relationships)) {
-                    // No need to compare nested documents more than max depth.
-                    if (count($this->relationshipWriteStack) >= Database::RELATION_MAX_DEPTH - 1) {
-                        continue;
-                    }
-                    $relationType = (string) $relationships[$key]['options']['relationType'];
-                    $side = (string) $relationships[$key]['options']['side'];
-                    switch ($relationType) {
-                        case Database::RELATION_ONE_TO_ONE:
-                            $oldValue = $old->getAttribute($key) instanceof Document
-                                ? $old->getAttribute($key)->getId()
-                                : $old->getAttribute($key);
-
-                            if ((\is_null($value) !== \is_null($oldValue))
-                                || (\is_string($value) && $value !== $oldValue)
-                                || ($value instanceof Document && $value->getId() !== $oldValue)
-                            ) {
-                                $shouldUpdate = true;
-                            }
-                            break;
-                        case Database::RELATION_ONE_TO_MANY:
-                        case Database::RELATION_MANY_TO_ONE:
-                        case Database::RELATION_MANY_TO_MANY:
-                            if (
-                                ($relationType === Database::RELATION_MANY_TO_ONE && $side === Database::RELATION_SIDE_PARENT) ||
-                                ($relationType === Database::RELATION_ONE_TO_MANY && $side === Database::RELATION_SIDE_CHILD)
-                            ) {
+                // Compare if the document has any changes
+                foreach ($document as $key => $value) {
+                    // Skip the nested documents as they will be checked later in recursions.
+                    if (\array_key_exists($key, $relationships)) {
+                        // No need to compare nested documents more than max depth.
+                        if (count($this->relationshipWriteStack) >= Database::RELATION_MAX_DEPTH - 1) {
+                            continue;
+                        }
+                        $relationType = (string)$relationships[$key]['options']['relationType'];
+                        $side = (string)$relationships[$key]['options']['side'];
+                        switch ($relationType) {
+                            case Database::RELATION_ONE_TO_ONE:
                                 $oldValue = $old->getAttribute($key) instanceof Document
                                     ? $old->getAttribute($key)->getId()
                                     : $old->getAttribute($key);
 
                                 if ((\is_null($value) !== \is_null($oldValue))
                                     || (\is_string($value) && $value !== $oldValue)
-                                    || ($value instanceof Document &&  $value->getId() !== $oldValue)
+                                    || ($value instanceof Document && $value->getId() !== $oldValue)
                                 ) {
                                     $shouldUpdate = true;
                                 }
                                 break;
-                            }
-
-                            if (!\is_array($value) || !\array_is_list($value)) {
-                                throw new RelationshipException('Invalid relationship value. Must be either an array of documents or document IDs, ' . \gettype($value) . ' given.');
-                            }
+                            case Database::RELATION_ONE_TO_MANY:
+                            case Database::RELATION_MANY_TO_ONE:
+                            case Database::RELATION_MANY_TO_MANY:
+                                if (
+                                    ($relationType === Database::RELATION_MANY_TO_ONE && $side === Database::RELATION_SIDE_PARENT) ||
+                                    ($relationType === Database::RELATION_ONE_TO_MANY && $side === Database::RELATION_SIDE_CHILD)
+                                ) {
+                                    $oldValue = $old->getAttribute($key) instanceof Document
+                                        ? $old->getAttribute($key)->getId()
+                                        : $old->getAttribute($key);
 
-                            if (\count($old->getAttribute($key)) !== \count($value)) {
-                                $shouldUpdate = true;
-                                break;
-                            }
+                                    if ((\is_null($value) !== \is_null($oldValue))
+                                        || (\is_string($value) && $value !== $oldValue)
+                                        || ($value instanceof Document && $value->getId() !== $oldValue)
+                                    ) {
+                                        $shouldUpdate = true;
+                                    }
+                                    break;
+                                }
 
-                            foreach ($value as $index => $relation) {
-                                $oldValue = $old->getAttribute($key)[$index] instanceof Document
-                                    ? $old->getAttribute($key)[$index]->getId()
-                                    : $old->getAttribute($key)[$index];
+                                if (!\is_array($value) || !\array_is_list($value)) {
+                                    throw new RelationshipException('Invalid relationship value. Must be either an array of documents or document IDs, ' . \gettype($value) . ' given.');
+                                }
 
-                                if (
-                                    (\is_string($relation) && $relation !== $oldValue) ||
-                                    ($relation instanceof Document && $relation->getId() !== $oldValue)
-                                ) {
+                                if (\count($old->getAttribute($key)) !== \count($value)) {
                                     $shouldUpdate = true;
                                     break;
                                 }
-                            }
+
+                                foreach ($value as $index => $relation) {
+                                    $oldValue = $old->getAttribute($key)[$index] instanceof Document
+                                        ? $old->getAttribute($key)[$index]->getId()
+                                        : $old->getAttribute($key)[$index];
+
+                                    if (
+                                        (\is_string($relation) && $relation !== $oldValue) ||
+                                        ($relation instanceof Document && $relation->getId() !== $oldValue)
+                                    ) {
+                                        $shouldUpdate = true;
+                                        break;
+                                    }
+                                }
+                                break;
+                        }
+
+                        if ($shouldUpdate) {
                             break;
+                        }
+
+                        continue;
                     }
 
-                    if ($shouldUpdate) {
+                    $oldValue = $old->getAttribute($key);
+
+                    // If values are not equal we need to update document.
+                    if ($value !== $oldValue) {
+                        $shouldUpdate = true;
                         break;
                     }
-
-                    continue;
                 }
 
-                $oldValue = $old->getAttribute($key);
+                $updatePermissions = [
+                    ...$collection->getUpdate(),
+                    ...($documentSecurity ? $old->getUpdate() : [])
+                ];
 
-                // If values are not equal we need to update document.
-                if ($value !== $oldValue) {
-                    $shouldUpdate = true;
-                    break;
+                $readPermissions = [
+                    ...$collection->getRead(),
+                    ...($documentSecurity ? $old->getRead() : [])
+                ];
+
+                if ($shouldUpdate && !$updateValidator->isValid($updatePermissions)) {
+                    throw new AuthorizationException($updateValidator->getDescription());
+                } elseif (!$shouldUpdate && !$readValidator->isValid($readPermissions)) {
+                    throw new AuthorizationException($readValidator->getDescription());
                 }
             }
 
-            $updatePermissions = [
-                ...$collection->getUpdate(),
-                ...($documentSecurity ? $old->getUpdate() : [])
-            ];
-
-            $readPermissions = [
-                ...$collection->getRead(),
-                ...($documentSecurity ? $old->getRead() : [])
-            ];
-
-            if ($shouldUpdate && !$updateValidator->isValid($updatePermissions)) {
-                throw new AuthorizationException($updateValidator->getDescription());
-            } elseif (!$shouldUpdate && !$readValidator->isValid($readPermissions)) {
-                throw new AuthorizationException($readValidator->getDescription());
+            if ($old->isEmpty()) {
+                return new Document();
             }
-        }
 
-        if ($old->isEmpty()) {
-            return new Document();
-        }
+            if ($shouldUpdate) {
+                $updatedAt = $document->getUpdatedAt();
+                $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
+            }
 
-        if ($shouldUpdate) {
-            $updatedAt = $document->getUpdatedAt();
-            $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
-        }
+            // Check if document was updated after the request timestamp
+            $oldUpdatedAt = new \DateTime($old->getUpdatedAt());
+            if (!is_null($this->timestamp) && $oldUpdatedAt > $this->timestamp) {
+                throw new ConflictException('Document was updated after the request timestamp');
+            }
 
-        // Check if document was updated after the request timestamp
-        $oldUpdatedAt = new \DateTime($old->getUpdatedAt());
-        if (!is_null($this->timestamp) && $oldUpdatedAt > $this->timestamp) {
-            throw new ConflictException('Document was updated after the request timestamp');
-        }
+            $document = $this->encode($collection, $document);
 
-        $document = $this->encode($collection, $document);
+            $structureValidator = new Structure($collection);
 
-        $structureValidator = new Structure($collection);
+            if (!$structureValidator->isValid($document)) { // Make sure updated structure still apply collection rules (if any)
+                throw new StructureException($structureValidator->getDescription());
+            }
 
-        if (!$structureValidator->isValid($document)) { // Make sure updated structure still apply collection rules (if any)
-            throw new StructureException($structureValidator->getDescription());
-        }
+            if ($this->resolveRelationships) {
+                $document = $this->silent(fn () => $this->updateDocumentRelationships($collection, $old, $document));
+            }
 
-        if ($this->resolveRelationships) {
-            $document = $this->silent(fn () => $this->updateDocumentRelationships($collection, $old, $document));
-        }
+            $this->adapter->updateDocument($collection->getId(), $document);
 
-        $this->adapter->updateDocument($collection->getId(), $document);
+            return $document;
+        });
 
         if ($this->resolveRelationships) {
             $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document));
@@ -3726,9 +3790,7 @@ public function updateDocument(string $collection, string $id, Document $documen
         $document = $this->decode($collection, $document);
 
         $this->purgeRelatedDocuments($collection, $id);
-
         $this->purgeCachedDocument($collection->getId(), $id);
-
         $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document);
 
         return $document;
@@ -3753,44 +3815,48 @@ public function updateDocuments(string $collection, array $documents, int $batch
             return [];
         }
 
-        $time = DateTime::now();
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
-        foreach ($documents as $key => $document) {
-            if (!$document->getId()) {
-                throw new DatabaseException('Must define $id attribute for each document');
-            }
+        $documents = $this->withTransaction(function () use ($collection, $documents, $batchSize) {
+            $time = DateTime::now();
 
-            $updatedAt = $document->getUpdatedAt();
-            $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
-            $document = $this->encode($collection, $document);
+            foreach ($documents as $key => $document) {
+                if (!$document->getId()) {
+                    throw new DatabaseException('Must define $id attribute for each document');
+                }
 
-            $old = Authorization::skip(fn () => $this->silent(
-                fn () => $this->getDocument(
-                    $collection->getId(),
-                    $document->getId()
-                )
-            ));
+                $updatedAt = $document->getUpdatedAt();
+                $document->setAttribute('$updatedAt', empty($updatedAt) || !$this->preserveDates ? $time : $updatedAt);
+                $document = $this->encode($collection, $document);
 
-            $validator = new Authorization(self::PERMISSION_UPDATE);
-            if (
-                $collection->getId() !== self::METADATA
-                && !$validator->isValid($old->getUpdate())
-            ) {
-                throw new AuthorizationException($validator->getDescription());
-            }
+                $old = Authorization::skip(fn () => $this->silent(
+                    fn () => $this->getDocument(
+                        $collection->getId(),
+                        $document->getId(),
+                        forUpdate: true
+                    )
+                ));
 
-            $validator = new Structure($collection);
-            if (!$validator->isValid($document)) {
-                throw new StructureException($validator->getDescription());
-            }
+                $validator = new Authorization(self::PERMISSION_UPDATE);
+                if (
+                    $collection->getId() !== self::METADATA
+                    && !$validator->isValid($old->getUpdate())
+                ) {
+                    throw new AuthorizationException($validator->getDescription());
+                }
 
-            if ($this->resolveRelationships) {
-                $documents[$key] = $this->silent(fn () => $this->updateDocumentRelationships($collection, $old, $document));
+                $validator = new Structure($collection);
+                if (!$validator->isValid($document)) {
+                    throw new StructureException($validator->getDescription());
+                }
+
+                if ($this->resolveRelationships) {
+                    $documents[$key] = $this->silent(fn () => $this->updateDocumentRelationships($collection, $old, $document));
+                }
             }
-        }
 
-        $documents = $this->adapter->updateDocuments($collection->getId(), $documents, $batchSize);
+            return $this->adapter->updateDocuments($collection->getId(), $documents, $batchSize);
+        });
 
         foreach ($documents as $key => $document) {
             if ($this->resolveRelationships) {
@@ -4394,34 +4460,41 @@ public function deleteDocument(string $collection, string $id): bool
 
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
-        $validator = new Authorization(self::PERMISSION_DELETE);
+        $deleted = $this->withTransaction(function () use ($collection, $id, &$document) {
+            $document = Authorization::skip(fn () => $this->silent(
+                fn () =>
+                $this->getDocument($collection->getId(), $id, forUpdate: true)
+            ));
 
-        if ($collection->getId() !== self::METADATA) {
-            $documentSecurity = $collection->getAttribute('documentSecurity', false);
-            if (!$validator->isValid([
-                ...$collection->getDelete(),
-                ...($documentSecurity ? $document->getDelete() : [])
-            ])) {
-                throw new AuthorizationException($validator->getDescription());
+            $validator = new Authorization(self::PERMISSION_DELETE);
+
+            if ($collection->getId() !== self::METADATA) {
+                $documentSecurity = $collection->getAttribute('documentSecurity', false);
+                if (!$validator->isValid([
+                    ...$collection->getDelete(),
+                    ...($documentSecurity ? $document->getDelete() : [])
+                ])) {
+                    throw new AuthorizationException($validator->getDescription());
+                }
             }
-        }
 
-        // Check if document was updated after the request timestamp
-        try {
-            $oldUpdatedAt = new \DateTime($document->getUpdatedAt());
-        } catch (Exception $e) {
-            throw new DatabaseException($e->getMessage(), $e->getCode(), $e);
-        }
+            // Check if document was updated after the request timestamp
+            try {
+                $oldUpdatedAt = new \DateTime($document->getUpdatedAt());
+            } catch (Exception $e) {
+                throw new DatabaseException($e->getMessage(), $e->getCode(), $e);
+            }
 
-        if (!is_null($this->timestamp) && $oldUpdatedAt > $this->timestamp) {
-            throw new ConflictException('Document was updated after the request timestamp');
-        }
+            if (!is_null($this->timestamp) && $oldUpdatedAt > $this->timestamp) {
+                throw new ConflictException('Document was updated after the request timestamp');
+            }
 
-        if ($this->resolveRelationships) {
-            $document = $this->silent(fn () => $this->deleteDocumentRelationships($collection, $document));
-        }
+            if ($this->resolveRelationships) {
+                $document = $this->silent(fn () => $this->deleteDocumentRelationships($collection, $document));
+            }
 
-        $deleted = $this->adapter->deleteDocument($collection->getId(), $id);
+            return $this->adapter->deleteDocument($collection->getId(), $id);
+        });
 
         $this->purgeRelatedDocuments($collection, $id);
         $this->purgeCachedDocument($collection->getId(), $id);

From 6eba4de1d4f10e83792f2eb2d3c730ae1c9cb2c5 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 30 Aug 2024 16:32:56 +1200
Subject: [PATCH 066/100] Merge pull request #442 from
 utopia-php/feat-resizing-string-attributes

Implement resizing string attributes
# Conflicts:
#	src/Database/Adapter/MariaDB.php
#	src/Database/Adapter/MySQL.php
#	src/Database/Adapter/Postgres.php
#	src/Database/Adapter/SQLite.php
---
 composer.lock                     | 34 +++++++--------
 docker-compose.yml                |  1 +
 src/Database/Adapter.php          |  7 ++++
 src/Database/Adapter/MariaDB.php  | 26 +++++++-----
 src/Database/Adapter/Mongo.php    |  5 +++
 src/Database/Adapter/MySQL.php    | 14 ++++---
 src/Database/Adapter/Postgres.php | 25 ++++++-----
 src/Database/Adapter/SQL.php      | 10 +++++
 src/Database/Adapter/SQLite.php   | 14 +++++--
 tests/e2e/Adapter/Base.php        | 70 ++++++++++++++++++++++++++++++-
 10 files changed, 158 insertions(+), 48 deletions(-)

diff --git a/composer.lock b/composer.lock
index 5bd08c299..b98610e74 100644
--- a/composer.lock
+++ b/composer.lock
@@ -898,35 +898,35 @@
         },
         {
             "name": "phpunit/php-code-coverage",
-            "version": "9.2.31",
+            "version": "9.2.32",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/php-code-coverage.git",
-                "reference": "48c34b5d8d983006bd2adc2d0de92963b9155965"
+                "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/48c34b5d8d983006bd2adc2d0de92963b9155965",
-                "reference": "48c34b5d8d983006bd2adc2d0de92963b9155965",
+                "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/85402a822d1ecf1db1096959413d35e1c37cf1a5",
+                "reference": "85402a822d1ecf1db1096959413d35e1c37cf1a5",
                 "shasum": ""
             },
             "require": {
                 "ext-dom": "*",
                 "ext-libxml": "*",
                 "ext-xmlwriter": "*",
-                "nikic/php-parser": "^4.18 || ^5.0",
+                "nikic/php-parser": "^4.19.1 || ^5.1.0",
                 "php": ">=7.3",
-                "phpunit/php-file-iterator": "^3.0.3",
-                "phpunit/php-text-template": "^2.0.2",
-                "sebastian/code-unit-reverse-lookup": "^2.0.2",
-                "sebastian/complexity": "^2.0",
-                "sebastian/environment": "^5.1.2",
-                "sebastian/lines-of-code": "^1.0.3",
-                "sebastian/version": "^3.0.1",
-                "theseer/tokenizer": "^1.2.0"
+                "phpunit/php-file-iterator": "^3.0.6",
+                "phpunit/php-text-template": "^2.0.4",
+                "sebastian/code-unit-reverse-lookup": "^2.0.3",
+                "sebastian/complexity": "^2.0.3",
+                "sebastian/environment": "^5.1.5",
+                "sebastian/lines-of-code": "^1.0.4",
+                "sebastian/version": "^3.0.2",
+                "theseer/tokenizer": "^1.2.3"
             },
             "require-dev": {
-                "phpunit/phpunit": "^9.3"
+                "phpunit/phpunit": "^9.6"
             },
             "suggest": {
                 "ext-pcov": "PHP extension that provides line coverage",
@@ -935,7 +935,7 @@
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "9.2-dev"
+                    "dev-main": "9.2.x-dev"
                 }
             },
             "autoload": {
@@ -964,7 +964,7 @@
             "support": {
                 "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues",
                 "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy",
-                "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.31"
+                "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/9.2.32"
             },
             "funding": [
                 {
@@ -972,7 +972,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2024-03-02T06:37:42+00:00"
+            "time": "2024-08-22T04:23:01+00:00"
         },
         {
             "name": "phpunit/php-file-iterator",
diff --git a/docker-compose.yml b/docker-compose.yml
index e784d8092..82076ebfe 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,6 +39,7 @@ services:
   mariadb:
     image: mariadb:10.11
     container_name: utopia-mariadb
+    command: mariadbd --max_allowed_packet=1G
     networks:
       - database
     ports:
diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php
index 05d651c8d..4295d42d5 100644
--- a/src/Database/Adapter.php
+++ b/src/Database/Adapter.php
@@ -738,6 +738,13 @@ abstract public function getSupportForRelationships(): bool;
 
     abstract public function getSupportForUpdateLock(): bool;
 
+    /**
+     * Is attribute resizing supported?
+     *
+     * @return bool
+     */
+    abstract public function getSupportForAttributeResizing(): bool;
+
     /**
      * Get current attribute count from collection document
      *
diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index b7ba2ff6c..e3dd72a09 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -336,10 +336,10 @@ public function updateAttribute(string $collection, string $id, string $type, in
     {
         $name = $this->filter($collection);
         $id = $this->filter($id);
+        $newKey = empty($newKey) ? null : $this->filter($newKey);
         $type = $this->getSQLType($type, $size, $signed, $array);
 
         if (!empty($newKey)) {
-            $newKey = $this->filter($newKey);
             $sql = "ALTER TABLE {$this->getSQLTable($name)} CHANGE COLUMN `{$id}` {$newKey} {$type};";
         } else {
             $sql = "ALTER TABLE {$this->getSQLTable($name)} MODIFY `{$id}` {$type};";
@@ -347,9 +347,13 @@ public function updateAttribute(string $collection, string $id, string $type, in
 
         $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql);
 
-        return $this->getPDO()
+        try {
+            return $this->getPDO()
             ->prepare($sql)
             ->execute();
+        } catch (PDOException $e) {
+            throw $this->processException($e);
+        }
     }
 
     /**
@@ -2241,28 +2245,28 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
 
     protected function processException(PDOException $e): \Exception
     {
-        /**
-         * PDO and Swoole PDOProxy swap error codes and errorInfo
-         */
-
         // Timeout
         if ($e->getCode() === '70100' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1969) {
             return new TimeoutException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 1969 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '70100') {
-            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
         // Duplicate table
         if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
         // Duplicate column
         if ($e->getCode() === '42S21' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1060) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 1060 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S21') {
+        }
+
+        // Data is too big for column resize
+        if ($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) {
+            return new DatabaseException('Resize would result in data truncation', $e->getCode(), $e);
+        }
+
+        // Duplicate index
+        if ($e->getCode() === '42000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1061) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
index 9e48c9a41..90d3ad310 100644
--- a/src/Database/Adapter/Mongo.php
+++ b/src/Database/Adapter/Mongo.php
@@ -1661,6 +1661,11 @@ public function getSupportForUpdateLock(): bool
         return false;
     }
 
+    public function getSupportForAttributeResizing(): bool
+    {
+        return false;
+    }
+
     /**
      * Get current attribute count from collection document
      *
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index 67be8477f..3503757c4 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -88,24 +88,28 @@ protected function processException(PDOException $e): \Exception
         // Timeout
         if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
             return new TimeoutException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 3024 && isset($e->errorInfo[0]) && $e->errorInfo[0] === "HY000") {
-            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
         // Duplicate table
         if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 1050 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S01') {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
         // Duplicate column
         if ($e->getCode() === '42S21' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1060) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 1060 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42S21') {
+        }
+
+        // Duplicate index
+        if ($e->getCode() === '42000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1061) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
+        // Data is too big for column resize
+        if ($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) {
+            return new DatabaseException('Resize would result in data truncation', $e->getCode(), $e);
+        }
+
         return $e;
     }
 }
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index edbb00203..6f145b9b7 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -281,6 +281,7 @@ public function deleteCollection(string $id): bool
      * @param bool $array
      *
      * @return bool
+     * @throws Exception
      */
     public function createAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool
     {
@@ -312,6 +313,7 @@ public function createAttribute(string $collection, string $id, string $type, in
      * @param bool $array
      *
      * @return bool
+     * @throws DatabaseException
      */
     public function deleteAttribute(string $collection, string $id, bool $array = false): bool
     {
@@ -410,11 +412,15 @@ public function updateAttribute(string $collection, string $id, string $type, in
 
         $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql);
 
-        $result = $this->getPDO()
+        try {
+            $result = $this->getPDO()
             ->prepare($sql)
             ->execute();
 
-        return $result;
+            return $result;
+        } catch (PDOException $e) {
+            throw $this->processException($e);
+        }
     }
 
     /**
@@ -2228,8 +2234,8 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
 			";
         });
     }
-
-    /**
+	
+	/**
      * @return string
      */
     public function getLikeOperator(): string
@@ -2242,22 +2248,21 @@ protected function processException(PDOException $e): \Exception
         // Timeout
         if ($e->getCode() === '57014' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
             return new TimeoutException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 7 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '57014') {
-            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
         // Duplicate table
         if ($e->getCode() === '42P07' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 7 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42P07') {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
         // Duplicate column
         if ($e->getCode() === '42701' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 7 && isset($e->errorInfo[0]) && $e->errorInfo[0] === '42701') {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        }
+
+        // Data is too big for column resize
+        if ($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
+            return new DatabaseException('Resize would result in data truncation', $e->getCode(), $e);
         }
 
         return $e;
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index e115afdb3..7b6967197 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -351,6 +351,16 @@ public function getSupportForUpdateLock(): bool
         return true;
     }
 
+    /**
+     * Is Attribute Resizing Supported?
+     *
+     * @return bool
+     */
+    public function getSupportForAttributeResizing(): bool
+    {
+        return true;
+    }
+
     /**
      * Get current attribute count from collection document
      *
diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index e9f4de36e..5ed5f9318 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -1095,6 +1095,16 @@ public function getSupportForUpdateLock(): bool
         return false;
     }
 
+    /**
+     * Is attribute resizing supported?
+     *
+     * @return bool
+     */
+    public function getSupportForAttributeResizing(): bool
+    {
+        return false;
+    }
+
     /**
      * Get SQL Index Type
      *
@@ -1366,15 +1376,11 @@ protected function processException(PDOException $e): \Exception
         // Timeout
         if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
             return new TimeoutException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 3024 && isset($e->errorInfo[0]) && $e->errorInfo[0] === "HY000") {
-            return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
         // Duplicate
         if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1) {
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        } elseif ($e->getCode() === 1 && isset($e->errorInfo[0]) && $e->errorInfo[0] === 'HY000') {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
         return $e;
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 9d97ec0bf..b462e4c4f 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -29,6 +29,8 @@
 use Utopia\Database\Validator\Structure;
 use Utopia\Validator\Range;
 
+ini_set('memory_limit', '2048M');
+
 abstract class Base extends TestCase
 {
     protected static string $namespace;
@@ -5863,6 +5865,72 @@ public function testUpdateAttributeRename(): void
         }
     }
 
+    public function createRandomString(int $length = 10): string
+    {
+        return \substr(\bin2hex(\random_bytes(\max(1, \intval(($length + 1) / 2)))), 0, $length);
+    }
+
+    public function updateStringAttributeSize(int $size, Document $document): Document
+    {
+        static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, $size, true);
+
+        $document = $document->setAttribute('resize_me', $this->createRandomString($size));
+
+        static::getDatabase()->updateDocument('resize_test', $document->getId(), $document);
+        $checkDoc = static::getDatabase()->getDocument('resize_test', $document->getId());
+
+        $this->assertEquals($document->getAttribute('resize_me'), $checkDoc->getAttribute('resize_me'));
+        $this->assertEquals($size, strlen($checkDoc->getAttribute('resize_me')));
+
+        return $checkDoc;
+    }
+
+    public function testUpdateAttributeSize(): void
+    {
+        if (!static::getDatabase()->getAdapter()->getSupportForAttributeResizing()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        static::getDatabase()->createCollection('resize_test');
+
+        $this->assertEquals(true, static::getDatabase()->createAttribute('resize_test', 'resize_me', Database::VAR_STRING, 128, true));
+        $document = static::getDatabase()->createDocument('resize_test', new Document([
+            '$id' => ID::unique(),
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::create(Role::any()),
+                Permission::update(Role::any()),
+                Permission::delete(Role::any()),
+            ],
+            'resize_me' => $this->createRandomString(128)
+        ]));
+
+        // Go up in size
+
+        // 0-16381 to 16382-65535
+        $document = $this->updateStringAttributeSize(16382, $document);
+
+        // 16382-65535 to 65536-16777215
+        $document = $this->updateStringAttributeSize(65536, $document);
+
+        // 65536-16777216 to PHP_INT_MAX or adapter limit
+        $maxStringSize = 16777217;
+        $document = $this->updateStringAttributeSize($maxStringSize, $document);
+
+        // Test going down in size with data that is too big (Expect Failure)
+        try {
+            static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, 128, true);
+            $this->fail('Succeeded updating attribute size to smaller size with data that is too big');
+        } catch (DatabaseException $e) {
+            $this->assertEquals('Resize would result in data truncation', $e->getMessage());
+        }
+
+        // Test going down in size when data isn't too big.
+        static::getDatabase()->updateDocument('resize_test', $document->getId(), $document->setAttribute('resize_me', $this->createRandomString(128)));
+        static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, 128, true);
+    }
+
     /**
      * @depends testCreatedAtUpdatedAt
      */
@@ -6060,7 +6128,7 @@ public function testKeywords(): void
             $this->assertEquals('reservedKeyDocument', $documents[0]->getId());
             $this->assertEquals('Reserved:' . $keyword, $documents[0]->getAttribute($keyword));
 
-            $documents = $database->find($collectionName, [Query::equal($keyword, ["Reserved:${keyword}"])]);
+            $documents = $database->find($collectionName, [Query::equal($keyword, ["Reserved:{$keyword}"])]);
             $this->assertCount(1, $documents);
             $this->assertEquals('reservedKeyDocument', $documents[0]->getId());
 

From 46fa933a0b69197950d0ff6705ff8f33c7f68983 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 30 Aug 2024 21:02:45 +1200
Subject: [PATCH 067/100] Merge pull request #444 from
 utopia-php/feat-resizing-string-attributes

Fix VARCHAR -> VARCHAR truncation and add tests
# Conflicts:
#	src/Database/Adapter/MariaDB.php
#	src/Database/Adapter/MySQL.php
---
 src/Database/Adapter/MariaDB.php |  3 ++-
 src/Database/Adapter/MySQL.php   |  3 ++-
 tests/e2e/Adapter/Base.php       | 11 +++++++++++
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index e3dd72a09..bc394ffbd 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -2261,7 +2261,8 @@ protected function processException(PDOException $e): \Exception
         }
 
         // Data is too big for column resize
-        if ($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) {
+        if (($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) ||
+            ($e->getCode() === '01000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1265)) {
             return new DatabaseException('Resize would result in data truncation', $e->getCode(), $e);
         }
 
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index 3503757c4..e2b7cc81d 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -106,7 +106,8 @@ protected function processException(PDOException $e): \Exception
         }
 
         // Data is too big for column resize
-        if ($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) {
+        if (($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) ||
+            ($e->getCode() === '01000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1265)) {
             return new DatabaseException('Resize would result in data truncation', $e->getCode(), $e);
         }
 
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index b462e4c4f..4248ea903 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -5929,6 +5929,17 @@ public function testUpdateAttributeSize(): void
         // Test going down in size when data isn't too big.
         static::getDatabase()->updateDocument('resize_test', $document->getId(), $document->setAttribute('resize_me', $this->createRandomString(128)));
         static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, 128, true);
+
+        // VARCHAR -> VARCHAR Truncation Test
+        static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, 1000, true);
+        static::getDatabase()->updateDocument('resize_test', $document->getId(), $document->setAttribute('resize_me', $this->createRandomString(1000)));
+
+        try {
+            static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, 128, true);
+            $this->fail('Succeeded updating attribute size to smaller size with data that is too big');
+        } catch (DatabaseException $e) {
+            $this->assertEquals('Resize would result in data truncation', $e->getMessage());
+        }
     }
 
     /**

From 6e2547aac0160b7937bedea522cbbd3a3bc23fd8 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 30 Aug 2024 22:14:51 +1200
Subject: [PATCH 068/100] Merge pull request #445 from
 utopia-php/feat-resizing-string-attributes

Add Truncate Error class for resizing string attributes
# Conflicts:
#	src/Database/Adapter/MariaDB.php
#	src/Database/Adapter/MySQL.php
#	src/Database/Adapter/Postgres.php
---
 src/Database/Adapter/MariaDB.php    | 3 ++-
 src/Database/Adapter/MySQL.php      | 3 ++-
 src/Database/Adapter/Postgres.php   | 1 +
 src/Database/Exception/Truncate.php | 9 +++++++++
 tests/e2e/Adapter/Base.php          | 7 +++----
 5 files changed, 17 insertions(+), 6 deletions(-)
 create mode 100644 src/Database/Exception/Truncate.php

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index bc394ffbd..d0b2c06b8 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -10,6 +10,7 @@
 use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
+use Utopia\Database\Exception\Truncate as TruncateException;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 
@@ -2263,7 +2264,7 @@ protected function processException(PDOException $e): \Exception
         // Data is too big for column resize
         if (($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) ||
             ($e->getCode() === '01000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1265)) {
-            return new DatabaseException('Resize would result in data truncation', $e->getCode(), $e);
+            return new TruncateException('Resize would result in data truncation', $e->getCode(), $e);
         }
 
         // Duplicate index
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index e2b7cc81d..9ddc683c3 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -7,6 +7,7 @@
 use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
+use Utopia\Database\Exception\Truncate as TruncateException;
 
 class MySQL extends MariaDB
 {
@@ -108,7 +109,7 @@ protected function processException(PDOException $e): \Exception
         // Data is too big for column resize
         if (($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) ||
             ($e->getCode() === '01000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1265)) {
-            return new DatabaseException('Resize would result in data truncation', $e->getCode(), $e);
+            return new TruncateException('Resize would result in data truncation', $e->getCode(), $e);
         }
 
         return $e;
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index 6f145b9b7..fcee00ec5 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -11,6 +11,7 @@
 use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
+use Utopia\Database\Exception\Truncate as TruncateException;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 
diff --git a/src/Database/Exception/Truncate.php b/src/Database/Exception/Truncate.php
new file mode 100644
index 000000000..9bd0ffb12
--- /dev/null
+++ b/src/Database/Exception/Truncate.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace Utopia\Database\Exception;
+
+use Utopia\Database\Exception;
+
+class Truncate extends Exception
+{
+}
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 4248ea903..302dfd223 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -19,6 +19,7 @@
 use Utopia\Database\Exception\Restricted as RestrictedException;
 use Utopia\Database\Exception\Structure as StructureException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
+use Utopia\Database\Exception\Truncate as TruncateException;
 use Utopia\Database\Helpers\ID;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
@@ -5922,8 +5923,7 @@ public function testUpdateAttributeSize(): void
         try {
             static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, 128, true);
             $this->fail('Succeeded updating attribute size to smaller size with data that is too big');
-        } catch (DatabaseException $e) {
-            $this->assertEquals('Resize would result in data truncation', $e->getMessage());
+        } catch (TruncateException $e) {
         }
 
         // Test going down in size when data isn't too big.
@@ -5937,8 +5937,7 @@ public function testUpdateAttributeSize(): void
         try {
             static::getDatabase()->updateAttribute('resize_test', 'resize_me', Database::VAR_STRING, 128, true);
             $this->fail('Succeeded updating attribute size to smaller size with data that is too big');
-        } catch (DatabaseException $e) {
-            $this->assertEquals('Resize would result in data truncation', $e->getMessage());
+        } catch (TruncateException $e) {
         }
     }
 

From 4be46b9528dc9cf92dc52853d830cfe9c210ad48 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 2 Sep 2024 18:28:50 +1200
Subject: [PATCH 069/100] Merge pull request #446 from utopia-php/fix-deadlock

Fix deadlock when selecting permissions for update
---
 src/Database/Adapter/MariaDB.php  | 8 +++-----
 src/Database/Adapter/Postgres.php | 2 --
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index d0b2c06b8..5ec3b9930 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -1074,8 +1074,6 @@ public function updateDocument(string $collection, Document $document): Document
                 $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
             }
 
-            $sql .= ' FOR UPDATE';
-
             $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
 
             /**
@@ -1231,9 +1229,9 @@ public function updateDocument(string $collection, Document $document): Document
             }
 
             $sql = "
-			UPDATE {$this->getSQLTable($name)}
-			SET {$columns} _uid = :_uid 
-			WHERE _uid = :_uid
+                UPDATE {$this->getSQLTable($name)}
+                SET {$columns} _uid = :_uid 
+                WHERE _uid = :_uid
 			";
 
             if ($this->sharedTables) {
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index fcee00ec5..3b338137e 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -1039,8 +1039,6 @@ public function updateDocument(string $collection, Document $document): Document
             $sql .= ' AND (_tenant = :_tenant OR _tenant IS NULL)';
         }
 
-        $sql .= ' FOR UPDATE';
-
         $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql);
 
         /**

From b6b7b54355648d799b3ea1bf6dbdd713bc026eb3 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 10 Sep 2024 22:08:37 +1200
Subject: [PATCH 070/100] Merge pull request #448 from
 utopia-php/fix-add-escapes-for-rename

Fix non-quoted newKey in MariaDB and add newKey filtering for postgres
# Conflicts:
#	composer.lock
---
 composer.lock                     | 56 +++++++++++++++++++++++++++----
 src/Database/Adapter/MariaDB.php  |  2 +-
 src/Database/Adapter/Postgres.php |  1 +
 tests/e2e/Adapter/Base.php        | 12 +++++++
 4 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/composer.lock b/composer.lock
index b98610e74..0f44d6979 100644
--- a/composer.lock
+++ b/composer.lock
@@ -136,20 +136,20 @@
         },
         {
             "name": "symfony/polyfill-php80",
-            "version": "v1.30.0",
+            "version": "v1.31.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/polyfill-php80.git",
-                "reference": "77fa7995ac1b21ab60769b7323d600a991a90433"
+                "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/77fa7995ac1b21ab60769b7323d600a991a90433",
-                "reference": "77fa7995ac1b21ab60769b7323d600a991a90433",
+                "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/60328e362d4c2c802a54fcbf04f9d3fb892b4cf8",
+                "reference": "60328e362d4c2c802a54fcbf04f9d3fb892b4cf8",
                 "shasum": ""
             },
             "require": {
-                "php": ">=7.1"
+                "php": ">=7.2"
             },
             "type": "library",
             "extra": {
@@ -196,7 +196,7 @@
                 "shim"
             ],
             "support": {
-                "source": "https://github.com/symfony/polyfill-php80/tree/v1.30.0"
+                "source": "https://github.com/symfony/polyfill-php80/tree/v1.31.0"
             },
             "funding": [
                 {
@@ -212,7 +212,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2024-05-31T15:07:36+00:00"
+            "time": "2024-09-09T11:45:10+00:00"
         },
         {
             "name": "utopia-php/cache",
@@ -269,11 +269,53 @@
             "version": "0.33.8",
             "source": {
                 "type": "git",
+                "url": "https://github.com/utopia-php/di.git",
+                "reference": "22490c95f7ac3898ed1c33f1b1b5dd577305ee31"
                 "url": "https://github.com/utopia-php/http.git",
                 "reference": "a7f577540a25cb90896fef2b64767bf8d700f3c5"
             },
             "dist": {
                 "type": "zip",
+                "url": "https://api.github.com/repos/utopia-php/di/zipball/22490c95f7ac3898ed1c33f1b1b5dd577305ee31",
+                "reference": "22490c95f7ac3898ed1c33f1b1b5dd577305ee31",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=8.2"
+            },
+            "require-dev": {
+                "laravel/pint": "^1.2",
+                "phpbench/phpbench": "^1.2",
+                "phpstan/phpstan": "^1.10",
+                "phpunit/phpunit": "^9.5.25",
+                "swoole/ide-helper": "4.8.3"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Utopia\\": "src/",
+                    "Tests\\E2E\\": "tests/e2e"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "description": "A simple and lite library for managing dependency injections",
+            "keywords": [
+                "framework",
+                "http",
+                "php",
+                "upf"
+            ],
+            "support": {
+                "issues": "https://github.com/utopia-php/di/issues",
+                "source": "https://github.com/utopia-php/di/tree/0.1.0"
+            },
+            "time": "2024-08-08T14:35:19+00:00"
+        },
+        {
+            "name": "utopia-php/framework",
                 "url": "https://api.github.com/repos/utopia-php/http/zipball/a7f577540a25cb90896fef2b64767bf8d700f3c5",
                 "reference": "a7f577540a25cb90896fef2b64767bf8d700f3c5",
                 "shasum": ""
diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 5ec3b9930..d4752bc0c 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -341,7 +341,7 @@ public function updateAttribute(string $collection, string $id, string $type, in
         $type = $this->getSQLType($type, $size, $signed, $array);
 
         if (!empty($newKey)) {
-            $sql = "ALTER TABLE {$this->getSQLTable($name)} CHANGE COLUMN `{$id}` {$newKey} {$type};";
+            $sql = "ALTER TABLE {$this->getSQLTable($name)} CHANGE COLUMN `{$id}` `{$newKey}` {$type};";
         } else {
             $sql = "ALTER TABLE {$this->getSQLTable($name)} MODIFY `{$id}` {$type};";
         }
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index 3b338137e..53187dd04 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -379,6 +379,7 @@ public function updateAttribute(string $collection, string $id, string $type, in
     {
         $name = $this->filter($collection);
         $id = $this->filter($id);
+        $newKey = empty($newKey) ? null : $this->filter($newKey);
         $type = $this->getSQLType($type, $size, $signed, $array);
 
         if ($type == 'TIMESTAMP(3)') {
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 302dfd223..7ae1ba50d 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -5864,6 +5864,18 @@ public function testUpdateAttributeRename(): void
         } catch (\Exception $e) {
             $this->assertInstanceOf(StructureException::class, $e);
         }
+
+        // Check new key filtering
+        static::getDatabase()->updateAttribute(
+            collection: 'rename_test',
+            id: 'renamed',
+            newKey: 'renamed-test',
+        );
+
+        $doc = static::getDatabase()->getDocument('rename_test', $doc->getId());
+
+        $this->assertEquals('string', $doc->getAttribute('renamed-test'));
+        $this->assertArrayNotHasKey('renamed', $doc->getAttributes());
     }
 
     public function createRandomString(int $length = 10): string

From 32ab2f79315906053586513a3e8f994cb8f783e5 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 12 Sep 2024 23:01:10 +1200
Subject: [PATCH 071/100] Lint

---
 composer.lock                           | 58 ++++---------------------
 src/Database/Adapter/MariaDB.php        | 22 +++++-----
 src/Database/Adapter/Mongo.php          |  7 ++-
 src/Database/Adapter/Postgres.php       | 27 +++++-------
 src/Database/Adapter/SQL.php            |  6 +--
 src/Database/Adapter/SQLite.php         | 12 ++---
 src/Database/Database.php               |  2 +-
 src/Database/Exception.php              |  2 +-
 src/Database/Query.php                  |  4 +-
 src/Database/Validator/Datetime.php     | 18 ++++----
 src/Database/Validator/Index.php        | 14 +++---
 src/Database/Validator/Queries.php      |  4 +-
 src/Database/Validator/Query/Filter.php | 18 ++++----
 src/Database/Validator/Structure.php    |  2 +-
 tests/e2e/Adapter/Base.php              | 42 +++++++++---------
 tests/unit/Validator/DateTimeTest.php   |  2 +-
 16 files changed, 97 insertions(+), 143 deletions(-)

diff --git a/composer.lock b/composer.lock
index 0f44d6979..32455f27d 100644
--- a/composer.lock
+++ b/composer.lock
@@ -269,53 +269,11 @@
             "version": "0.33.8",
             "source": {
                 "type": "git",
-                "url": "https://github.com/utopia-php/di.git",
-                "reference": "22490c95f7ac3898ed1c33f1b1b5dd577305ee31"
                 "url": "https://github.com/utopia-php/http.git",
                 "reference": "a7f577540a25cb90896fef2b64767bf8d700f3c5"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/di/zipball/22490c95f7ac3898ed1c33f1b1b5dd577305ee31",
-                "reference": "22490c95f7ac3898ed1c33f1b1b5dd577305ee31",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=8.2"
-            },
-            "require-dev": {
-                "laravel/pint": "^1.2",
-                "phpbench/phpbench": "^1.2",
-                "phpstan/phpstan": "^1.10",
-                "phpunit/phpunit": "^9.5.25",
-                "swoole/ide-helper": "4.8.3"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "Utopia\\": "src/",
-                    "Tests\\E2E\\": "tests/e2e"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "description": "A simple and lite library for managing dependency injections",
-            "keywords": [
-                "framework",
-                "http",
-                "php",
-                "upf"
-            ],
-            "support": {
-                "issues": "https://github.com/utopia-php/di/issues",
-                "source": "https://github.com/utopia-php/di/tree/0.1.0"
-            },
-            "time": "2024-08-08T14:35:19+00:00"
-        },
-        {
-            "name": "utopia-php/framework",
                 "url": "https://api.github.com/repos/utopia-php/http/zipball/a7f577540a25cb90896fef2b64767bf8d700f3c5",
                 "reference": "a7f577540a25cb90896fef2b64767bf8d700f3c5",
                 "shasum": ""
@@ -548,16 +506,16 @@
         },
         {
             "name": "laravel/pint",
-            "version": "v1.17.2",
+            "version": "v1.17.3",
             "source": {
                 "type": "git",
                 "url": "https://github.com/laravel/pint.git",
-                "reference": "e8a88130a25e3f9d4d5785e6a1afca98268ab110"
+                "reference": "9d77be916e145864f10788bb94531d03e1f7b482"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/laravel/pint/zipball/e8a88130a25e3f9d4d5785e6a1afca98268ab110",
-                "reference": "e8a88130a25e3f9d4d5785e6a1afca98268ab110",
+                "url": "https://api.github.com/repos/laravel/pint/zipball/9d77be916e145864f10788bb94531d03e1f7b482",
+                "reference": "9d77be916e145864f10788bb94531d03e1f7b482",
                 "shasum": ""
             },
             "require": {
@@ -568,13 +526,13 @@
                 "php": "^8.1.0"
             },
             "require-dev": {
-                "friendsofphp/php-cs-fixer": "^3.61.1",
-                "illuminate/view": "^10.48.18",
+                "friendsofphp/php-cs-fixer": "^3.64.0",
+                "illuminate/view": "^10.48.20",
                 "larastan/larastan": "^2.9.8",
                 "laravel-zero/framework": "^10.4.0",
                 "mockery/mockery": "^1.6.12",
                 "nunomaduro/termwind": "^1.15.1",
-                "pestphp/pest": "^2.35.0"
+                "pestphp/pest": "^2.35.1"
             },
             "bin": [
                 "builds/pint"
@@ -610,7 +568,7 @@
                 "issues": "https://github.com/laravel/pint/issues",
                 "source": "https://github.com/laravel/pint"
             },
-            "time": "2024-08-06T15:11:54+00:00"
+            "time": "2024-09-03T15:00:28+00:00"
         },
         {
             "name": "myclabs/deep-copy",
diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index d4752bc0c..66f165550 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -705,7 +705,7 @@ public function createIndex(string $collection, string $id, string $type, array
 
             $attributes[$i] = "`{$attr}`{$length} {$order}";
 
-            if(!empty($collectionAttribute['array']) && $this->castIndexArray()) {
+            if (!empty($collectionAttribute['array']) && $this->castIndexArray()) {
                 $attributes[$i] = '(CAST(' . $attr . ' AS char(' . Database::ARRAY_INDEX_LENGTH . ') ARRAY))';
             }
         }
@@ -883,7 +883,7 @@ public function createDocument(string $collection, Document $document): Document
                 $stmtPermissions->execute();
             }
         } catch (\Throwable $e) {
-            if($e instanceof PDOException) {
+            if ($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
                     case 23000:
@@ -933,7 +933,7 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $attributes['_updatedAt'] = $document->getUpdatedAt();
                     $attributes['_permissions'] = \json_encode($document->getPermissions());
 
-                    if(!empty($document->getInternalId())) {
+                    if (!empty($document->getInternalId())) {
                         $internalIds[$document->getId()] = true;
                         $attributes['_id'] = $document->getInternalId();
                     }
@@ -1014,7 +1014,7 @@ public function createDocuments(string $collection, array $documents, int $batch
                 }
             }
         } catch (\Throwable $e) {
-            if($e instanceof PDOException) {
+            if ($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
                     case 23000:
@@ -1026,7 +1026,7 @@ public function createDocuments(string $collection, array $documents, int $batch
         }
 
         foreach ($documents as $document) {
-            if(!isset($internalIds[$document->getId()])) {
+            if (!isset($internalIds[$document->getId()])) {
                 $document['$internalId'] = $this->getDocument(
                     $collection,
                     $document->getId(),
@@ -1270,7 +1270,7 @@ public function updateDocument(string $collection, Document $document): Document
             }
 
         } catch (\Throwable $e) {
-            if($e instanceof PDOException) {
+            if ($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
                     case 23000:
@@ -1527,7 +1527,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                 }
             }
         } catch (\Throwable $e) {
-            if($e instanceof PDOException) {
+            if ($e instanceof PDOException) {
                 switch ($e->getCode()) {
                     case 1062:
                     case 23000:
@@ -1757,7 +1757,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         }
 
         $conditions = $this->getSQLConditions($queries);
-        if(!empty($conditions)) {
+        if (!empty($conditions)) {
             $where[] = $conditions;
         }
 
@@ -1883,7 +1883,7 @@ public function count(string $collection, array $queries = [], ?int $max = null)
         $limit = \is_null($max) ? '' : 'LIMIT :max';
 
         $conditions = $this->getSQLConditions($queries);
-        if(!empty($conditions)) {
+        if (!empty($conditions)) {
             $where[] = $conditions;
         }
 
@@ -2098,7 +2098,7 @@ protected function getSQLCondition(Query $query): string
                 return "`table_main`.{$attribute} {$this->getSQLOperator($query->getMethod())}";
 
             case Query::TYPE_CONTAINS:
-                if($this->getSupportForJSONOverlaps() && $query->onArray()) {
+                if ($this->getSupportForJSONOverlaps() && $query->onArray()) {
                     return "JSON_OVERLAPS(`table_main`.{$attribute}, :{$placeholder}_0)";
                 }
 
@@ -2124,7 +2124,7 @@ protected function getSQLCondition(Query $query): string
      */
     protected function getSQLType(string $type, int $size, bool $signed = true, bool $array = false): string
     {
-        if($array === true) {
+        if ($array === true) {
             return 'JSON';
         }
 
diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
index 90d3ad310..c4e20110f 100644
--- a/src/Database/Adapter/Mongo.php
+++ b/src/Database/Adapter/Mongo.php
@@ -3,7 +3,6 @@
 namespace Utopia\Database\Adapter;
 
 use Exception;
-
 use MongoDB\BSON\ObjectId;
 use MongoDB\BSON\Regex;
 use MongoDB\BSON\UTCDateTime;
@@ -291,7 +290,7 @@ public function getSizeOfCollection(string $collection): int
             } else {
                 throw new DatabaseException('No size found');
             }
-        } catch(Exception $e) {
+        } catch (Exception $e) {
             throw new DatabaseException('Failed to get collection size: ' . $e->getMessage());
         }
     }
@@ -1364,7 +1363,7 @@ protected function buildFilters(array $queries, string $separator = '$and'): arr
         $queries = Query::groupByType($queries)['filters'];
         foreach ($queries as $query) {
             /* @var $query Query */
-            if($query->isNested()) {
+            if ($query->isNested()) {
                 $operator = $this->getQueryOperator($query->getMethod());
                 $filters[$separator][] = $this->buildFilters($query->getValues(), $operator);
             } else {
@@ -1418,7 +1417,7 @@ protected function buildFilter(Query $query): array
         } elseif ($operator == '$ne' && \is_array($value)) {
             $filter[$attribute]['$nin'] = $value;
         } elseif ($operator == '$in') {
-            if($query->getMethod() === Query::TYPE_CONTAINS && !$query->onArray()) {
+            if ($query->getMethod() === Query::TYPE_CONTAINS && !$query->onArray()) {
                 $filter[$attribute]['$regex'] = new Regex(".*{$this->escapeWildcards($value)}.*", 'i');
             } else {
                 $filter[$attribute]['$in'] = $query->getValues();
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index 53187dd04..521e58d40 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -11,7 +11,6 @@
 use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
-use Utopia\Database\Exception\Truncate as TruncateException;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
 
@@ -864,7 +863,7 @@ public function createDocument(string $collection, Document $document): Document
 
             $queryPermissions = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $queryPermissions);
             $stmtPermissions = $this->getPDO()->prepare($queryPermissions);
-            if($sqlTenant) {
+            if ($sqlTenant) {
                 $stmtPermissions->bindValue(':_tenant', $this->tenant);
             }
         }
@@ -880,7 +879,7 @@ public function createDocument(string $collection, Document $document): Document
         } catch (Throwable $e) {
             switch ($e->getCode()) {
                 case 23505:
-                    throw new Duplicate('Duplicated document: ' . $e->getMessage());
+                    throw new DuplicateException('Duplicated document: ' . $e->getMessage());
                 default:
                     throw $e;
             }
@@ -924,12 +923,12 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $attributes['_updatedAt'] = $document->getUpdatedAt();
                     $attributes['_permissions'] = \json_encode($document->getPermissions());
 
-                    if(!empty($document->getInternalId())) {
+                    if (!empty($document->getInternalId())) {
                         $internalIds[$document->getId()] = true;
                         $attributes['_id'] = $document->getInternalId();
                     }
 
-                    if($this->sharedTables) {
+                    if ($this->sharedTables) {
                         $attributes['_tenant'] = $this->tenant;
                     }
 
@@ -984,8 +983,6 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $stmtPermissions?->execute();
                 }
             }
-
-            return $documents;
         } catch (PDOException $e) {
             throw match ($e->getCode()) {
                 1062, 23000 => new DuplicateException('Duplicated document: ' . $e->getMessage()),
@@ -994,7 +991,7 @@ public function createDocuments(string $collection, array $documents, int $batch
         }
 
         foreach ($documents as $document) {
-            if(!isset($internalIds[$document->getId()])) {
+            if (!isset($internalIds[$document->getId()])) {
                 $document['$internalId'] = $this->getDocument(
                     $collection,
                     $document->getId(),
@@ -1271,7 +1268,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     $attributes['_updatedAt'] = $document->getUpdatedAt();
                     $attributes['_permissions'] = json_encode($document->getPermissions());
 
-                    if($this->sharedTables) {
+                    if ($this->sharedTables) {
                         $attributes['_tenant'] = $this->tenant;
                     }
 
@@ -1463,7 +1460,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                         $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value));
                     }
 
-                    if($this->sharedTables) {
+                    if ($this->sharedTables) {
                         $stmtAddPermissions->bindValue(':_tenant', $this->tenant);
                     }
 
@@ -1697,7 +1694,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
         }
 
         $conditions = $this->getSQLConditions($queries);
-        if(!empty($conditions)) {
+        if (!empty($conditions)) {
             $where[] = $conditions;
         }
 
@@ -1823,7 +1820,7 @@ public function count(string $collection, array $queries = [], ?int $max = null)
         $limit = \is_null($max) ? '' : 'LIMIT :max';
 
         $conditions = $this->getSQLConditions($queries);
-        if(!empty($conditions)) {
+        if (!empty($conditions)) {
             $where[] = $conditions;
         }
 
@@ -2061,7 +2058,7 @@ protected function getFulltextValue(string $value): string
      */
     protected function getSQLType(string $type, int $size, bool $signed = true, bool $array = false): string
     {
-        if($array === true) {
+        if ($array === true) {
             return 'JSONB';
         }
 
@@ -2234,8 +2231,8 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL
 			";
         });
     }
-	
-	/**
+
+    /**
      * @return string
      */
     public function getLikeOperator(): string
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index 7b6967197..d7bce4f3a 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -818,14 +818,14 @@ protected function bindConditionValue(mixed $stmt, Query $query): void
             return;
         }
 
-        if($query->isNested()) {
+        if ($query->isNested()) {
             foreach ($query->getValues() as $value) {
                 $this->bindConditionValue($stmt, $value);
             }
             return;
         }
 
-        if($this->getSupportForJSONOverlaps() && $query->onArray() && $query->getMethod() == Query::TYPE_CONTAINS) {
+        if ($this->getSupportForJSONOverlaps() && $query->onArray() && $query->getMethod() == Query::TYPE_CONTAINS) {
             $placeholder = $this->getSQLPlaceholder($query) . '_0';
             $stmt->bindValue($placeholder, json_encode($query->getValues()), PDO::PARAM_STR);
             return;
@@ -1071,7 +1071,7 @@ public function getSQLConditions(array $queries = [], string $separator = 'AND')
                 continue;
             }
 
-            if($query->isNested()) {
+            if ($query->isNested()) {
                 $conditions[] = $this->getSQLConditions($query->getValues(), $query->getMethod());
             } else {
                 $conditions[] = $this->getSQLCondition($query);
diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index 5ed5f9318..08c3baa7b 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -459,7 +459,7 @@ public function createDocument(string $collection, Document $document): Document
         $attributes['_updatedAt'] = $document->getUpdatedAt();
         $attributes['_permissions'] = json_encode($document->getPermissions());
 
-        if($this->sharedTables) {
+        if ($this->sharedTables) {
             $attributes['_tenant'] = $this->tenant;
         }
 
@@ -533,7 +533,7 @@ public function createDocument(string $collection, Document $document): Document
 
             $stmtPermissions = $this->getPDO()->prepare($queryPermissions);
 
-            if($this->sharedTables) {
+            if ($this->sharedTables) {
                 $stmtPermissions->bindValue(':_tenant', $this->tenant);
             }
         }
@@ -578,7 +578,7 @@ public function updateDocument(string $collection, Document $document): Document
         $attributes['_updatedAt'] = $document->getUpdatedAt();
         $attributes['_permissions'] = json_encode($document->getPermissions());
 
-        if($this->sharedTables) {
+        if ($this->sharedTables) {
             $attributes['_tenant'] = $this->tenant;
         }
 
@@ -712,7 +712,7 @@ public function updateDocument(string $collection, Document $document): Document
             $stmtAddPermissions = $this->getPDO()->prepare($sql);
 
             $stmtAddPermissions->bindValue(":_uid", $document->getId());
-            if($this->sharedTables) {
+            if ($this->sharedTables) {
                 $stmtAddPermissions->bindValue(":_tenant", $this->tenant);
             }
 
@@ -825,7 +825,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     $attributes['_updatedAt'] = $document->getUpdatedAt();
                     $attributes['_permissions'] = json_encode($document->getPermissions());
 
-                    if($this->sharedTables) {
+                    if ($this->sharedTables) {
                         $attributes['_tenant'] = $this->tenant;
                     }
 
@@ -1023,7 +1023,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
                         $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value));
                     }
 
-                    if($this->sharedTables) {
+                    if ($this->sharedTables) {
                         $stmtAddPermissions->bindValue(':_tenant', $this->tenant);
                     }
 
diff --git a/src/Database/Database.php b/src/Database/Database.php
index 61b2f2b98..5421cd895 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -5048,7 +5048,7 @@ public function find(string $collection, array $queries = []): array
 
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
-        foreach ($results as  &$node) {
+        foreach ($results as &$node) {
             if ($this->resolveRelationships && (empty($selects) || !empty($nestedSelections))) {
                 $node = $this->silent(fn () => $this->populateDocumentRelationships($collection, $node, $nestedSelections));
             }
diff --git a/src/Database/Exception.php b/src/Database/Exception.php
index 64ad3c997..94099c6ae 100644
--- a/src/Database/Exception.php
+++ b/src/Database/Exception.php
@@ -8,7 +8,7 @@ class Exception extends \Exception
 {
     public function __construct(string $message, int|string $code = 0, Throwable $previous = null)
     {
-        if(\is_string($code)) {
+        if (\is_string($code)) {
             if (\is_numeric($code)) {
                 $code = (int) $code;
             } else {
diff --git a/src/Database/Query.php b/src/Database/Query.php
index b01534aab..8e9e574d5 100644
--- a/src/Database/Query.php
+++ b/src/Database/Query.php
@@ -288,7 +288,7 @@ public function toArray(): array
     {
         $array = ['method' => $this->method];
 
-        if(!empty($this->attribute)) {
+        if (!empty($this->attribute)) {
             $array['attribute'] = $this->attribute;
         }
 
@@ -681,7 +681,7 @@ public static function groupByType(array $queries): array
      */
     public function isNested(): bool
     {
-        if(in_array($this->getMethod(), self::LOGICAL_TYPES)) {
+        if (in_array($this->getMethod(), self::LOGICAL_TYPES)) {
             return true;
         }
 
diff --git a/src/Database/Validator/Datetime.php b/src/Database/Validator/Datetime.php
index e8ffee5ff..812669d7a 100644
--- a/src/Database/Validator/Datetime.php
+++ b/src/Database/Validator/Datetime.php
@@ -33,7 +33,7 @@ class Datetime extends Validator
      */
     public function __construct(bool $requireDateInFuture = false, string $precision = self::PRECISION_ANY, int $offset = 0)
     {
-        if($offset < 0) {
+        if ($offset < 0) {
             throw new \Exception('Offset must be a positive number.');
         }
 
@@ -50,13 +50,13 @@ public function getDescription(): string
     {
         $message = 'Value must be valid date';
 
-        if($this->offset > 0) {
+        if ($this->offset > 0) {
             $message .= " at least " . $this->offset . " seconds in future";
-        } elseif($this->requireDateInFuture) {
+        } elseif ($this->requireDateInFuture) {
             $message .= " in future";
         }
 
-        if($this->precision !== self::PRECISION_ANY) {
+        if ($this->precision !== self::PRECISION_ANY) {
             $message .= " with " . $this->precision . " precision";
         }
 
@@ -84,9 +84,9 @@ public function isValid($value): bool
                 return false;
             }
 
-            if($this->offset !== 0) {
+            if ($this->offset !== 0) {
                 $diff = $date->getTimestamp() - $now->getTimestamp();
-                if($diff <= $this->offset) {
+                if ($diff <= $this->offset) {
                     return false;
                 }
             }
@@ -109,12 +109,12 @@ public function isValid($value): bool
                     break;
             }
 
-            foreach($denyConstants as $constant) {
-                if(\intval($date->format($constant)) !== 0) {
+            foreach ($denyConstants as $constant) {
+                if (\intval($date->format($constant)) !== 0) {
                     return false;
                 }
             }
-        } catch(\Exception $e) {
+        } catch (\Exception $e) {
             return false;
         }
 
diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php
index b24e39f73..fdaa1efe0 100644
--- a/src/Database/Validator/Index.php
+++ b/src/Database/Validator/Index.php
@@ -126,30 +126,30 @@ public function checkArrayIndex(Document $index): bool
         foreach ($attributes as $attributePosition => $attributeName) {
             $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document();
 
-            if($attribute->getAttribute('array', false)) {
+            if ($attribute->getAttribute('array', false)) {
                 // Database::INDEX_UNIQUE Is not allowed! since mariaDB VS MySQL makes the unique Different on values
-                if($index->getAttribute('type') != Database::INDEX_KEY) {
+                if ($index->getAttribute('type') != Database::INDEX_KEY) {
                     $this->message = '"' . ucfirst($index->getAttribute('type')) . '" index is forbidden on array attributes';
                     return false;
                 }
 
-                if(empty($lengths[$attributePosition])) {
+                if (empty($lengths[$attributePosition])) {
                     $this->message = 'Index length for array not specified';
                     return false;
                 }
 
                 $arrayAttributes[] = $attribute->getAttribute('key', '');
-                if(count($arrayAttributes) > 1) {
+                if (count($arrayAttributes) > 1) {
                     $this->message = 'An index may only contain one array attribute';
                     return false;
                 }
 
                 $direction = $orders[$attributePosition] ?? '';
-                if(!empty($direction)) {
+                if (!empty($direction)) {
                     $this->message = 'Invalid index order "' . $direction . '" on array attribute "'. $attribute->getAttribute('key', '') .'"';
                     return false;
                 }
-            } elseif($attribute->getAttribute('type') !== Database::VAR_STRING && !empty($lengths[$attributePosition])) {
+            } elseif ($attribute->getAttribute('type') !== Database::VAR_STRING && !empty($lengths[$attributePosition])) {
                 $this->message = 'Cannot set a length on "'. $attribute->getAttribute('type') . '" attributes';
                 return false;
             }
@@ -188,7 +188,7 @@ public function checkIndexLength(Document $index): bool
                     break;
             }
 
-            if($attribute->getAttribute('array', false)) {
+            if ($attribute->getAttribute('array', false)) {
                 $attributeSize = Database::ARRAY_INDEX_LENGTH;
                 $indexLength = Database::ARRAY_INDEX_LENGTH;
             }
diff --git a/src/Database/Validator/Queries.php b/src/Database/Validator/Queries.php
index 2e4aac71a..b1d67aad0 100644
--- a/src/Database/Validator/Queries.php
+++ b/src/Database/Validator/Queries.php
@@ -71,8 +71,8 @@ public function isValid($value): bool
                 }
             }
 
-            if($query->isNested()) {
-                if(!self::isValid($query->getValues())) {
+            if ($query->isNested()) {
+                if (!self::isValid($query->getValues())) {
                     return false;
                 }
             }
diff --git a/src/Database/Validator/Query/Filter.php b/src/Database/Validator/Query/Filter.php
index f5027e6c5..4767d545c 100644
--- a/src/Database/Validator/Query/Filter.php
+++ b/src/Database/Validator/Query/Filter.php
@@ -132,29 +132,29 @@ protected function isValidAttributeAndValues(string $attribute, array $values, s
             }
         }
 
-        if($attributeSchema['type'] === 'relationship') {
+        if ($attributeSchema['type'] === 'relationship') {
             /**
              * We can not disable relationship query since we have logic that use it,
              * so instead we validate against the relation type
              */
             $options = $attributeSchema['options'];
 
-            if($options['relationType'] === Database::RELATION_ONE_TO_ONE && $options['twoWay'] === false && $options['side'] === Database::RELATION_SIDE_CHILD) {
+            if ($options['relationType'] === Database::RELATION_ONE_TO_ONE && $options['twoWay'] === false && $options['side'] === Database::RELATION_SIDE_CHILD) {
                 $this->message = 'Cannot query on virtual relationship attribute';
                 return false;
             }
 
-            if($options['relationType'] === Database::RELATION_ONE_TO_MANY && $options['side'] === Database::RELATION_SIDE_PARENT) {
+            if ($options['relationType'] === Database::RELATION_ONE_TO_MANY && $options['side'] === Database::RELATION_SIDE_PARENT) {
                 $this->message = 'Cannot query on virtual relationship attribute';
                 return false;
             }
 
-            if($options['relationType'] === Database::RELATION_MANY_TO_ONE && $options['side'] === Database::RELATION_SIDE_CHILD) {
+            if ($options['relationType'] === Database::RELATION_MANY_TO_ONE && $options['side'] === Database::RELATION_SIDE_CHILD) {
                 $this->message = 'Cannot query on virtual relationship attribute';
                 return false;
             }
 
-            if($options['relationType'] === Database::RELATION_MANY_TO_MANY) {
+            if ($options['relationType'] === Database::RELATION_MANY_TO_MANY) {
                 $this->message = 'Cannot query on virtual relationship attribute';
                 return false;
             }
@@ -162,7 +162,7 @@ protected function isValidAttributeAndValues(string $attribute, array $values, s
 
         $array = $attributeSchema['array'] ?? false;
 
-        if(
+        if (
             !$array &&
             $method === Query::TYPE_CONTAINS &&
             $attributeSchema['type'] !==  Database::VAR_STRING
@@ -171,7 +171,7 @@ protected function isValidAttributeAndValues(string $attribute, array $values, s
             return false;
         }
 
-        if(
+        if (
             $array &&
             !in_array($method, [Query::TYPE_CONTAINS, Query::TYPE_IS_NULL, Query::TYPE_IS_NOT_NULL])
         ) {
@@ -254,12 +254,12 @@ public function isValid($value): bool
             case Query::TYPE_AND:
                 $filters = Query::groupByType($value->getValues())['filters'];
 
-                if(count($value->getValues()) !== count($filters)) {
+                if (count($value->getValues()) !== count($filters)) {
                     $this->message = \ucfirst($method) . ' queries can only contain filter queries';
                     return false;
                 }
 
-                if(count($filters) < 2) {
+                if (count($filters) < 2) {
                     $this->message = \ucfirst($method) . ' queries require at least two queries';
                     return false;
                 }
diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php
index 12b753824..991a32240 100644
--- a/src/Database/Validator/Structure.php
+++ b/src/Database/Validator/Structure.php
@@ -257,7 +257,7 @@ public function isValid($document): bool
                 continue;
             }
 
-            if($type === Database::VAR_RELATIONSHIP) {
+            if ($type === Database::VAR_RELATIONSHIP) {
                 continue;
             }
 
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 7ae1ba50d..7d6852089 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -1028,7 +1028,7 @@ public function testQueryTimeout(): void
                 Query::notEqual('longtext', 'appwrite'),
             ]);
             $this->fail('Failed to throw exception');
-        } catch(\Exception $e) {
+        } catch (\Exception $e) {
             static::getDatabase()->clearTimeout();
             static::getDatabase()->deleteCollection('global-timeouts');
             $this->assertInstanceOf(TimeoutException::class, $e);
@@ -1869,7 +1869,7 @@ public function testCreateDocument(): Document
                 'empty' => [],
             ]));
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertTrue($e instanceof StructureException);
             $this->assertStringContainsString('Invalid document structure: Attribute "float_unsigned" has invalid type. Value must be a valid range between 0 and', $e->getMessage());
         }
@@ -1888,7 +1888,7 @@ public function testCreateDocument(): Document
                 'empty' => [],
             ]));
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertTrue($e instanceof StructureException);
             $this->assertEquals('Invalid document structure: Attribute "bigint_unsigned" has invalid type. Value must be a valid range between 0 and 9,223,372,036,854,775,807', $e->getMessage());
         }
@@ -2279,7 +2279,7 @@ public function testListDocumentSearch(): void
 
     public function testEmptyTenant(): void
     {
-        if(static::getDatabase()->getAdapter()->getSharedTables()) {
+        if (static::getDatabase()->getAdapter()->getSharedTables()) {
             $this->expectNotToPerformAssertions();
             return;
         }
@@ -2458,7 +2458,7 @@ public function testUpdateDocumentConflict(Document $document): void
                 return $this->getDatabase()->updateDocument($document->getCollection(), $document->getId(), $document);
             });
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertTrue($e instanceof ConflictException);
             $this->assertEquals('Document was updated after the request timestamp', $e->getMessage());
         }
@@ -2599,7 +2599,7 @@ public function testArrayAttribute(): void
         try {
             $database->createDocument($collection, new Document([]));
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('Invalid document structure: Missing required attribute "booleans"', $e->getMessage());
         }
 
@@ -2619,7 +2619,7 @@ public function testArrayAttribute(): void
                 'short' => ['More than 5 size'],
             ]));
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('Invalid document structure: Attribute "short[\'0\']" has invalid type. Value must be a valid string and no longer than 5 chars', $e->getMessage());
         }
 
@@ -2628,7 +2628,7 @@ public function testArrayAttribute(): void
                 'names' => ['Joe', 100],
             ]));
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('Invalid document structure: Attribute "names[\'1\']" has invalid type. Value must be a valid string and no longer than 255 chars', $e->getMessage());
         }
 
@@ -2637,7 +2637,7 @@ public function testArrayAttribute(): void
                 'age' => 1.5,
             ]));
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('Invalid document structure: Attribute "age" has invalid type. Value must be a valid integer', $e->getMessage());
         }
 
@@ -2646,7 +2646,7 @@ public function testArrayAttribute(): void
                 'age' => -100,
             ]));
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('Invalid document structure: Attribute "age" has invalid type. Value must be a valid range between 0 and 2,147,483,647', $e->getMessage());
         }
 
@@ -2675,7 +2675,7 @@ public function testArrayAttribute(): void
         try {
             $database->createIndex($collection, 'indx', Database::INDEX_FULLTEXT, ['names']);
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             if ($this->getDatabase()->getAdapter()->getSupportForFulltextIndex()) {
                 $this->assertEquals('"Fulltext" index is forbidden on array attributes', $e->getMessage());
             } else {
@@ -2686,7 +2686,7 @@ public function testArrayAttribute(): void
         try {
             $database->createIndex($collection, 'indx', Database::INDEX_KEY, ['numbers', 'names'], [100,100]);
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('An index may only contain one array attribute', $e->getMessage());
         }
 
@@ -2707,7 +2707,7 @@ public function testArrayAttribute(): void
             try {
                 $database->createIndex($collection, 'indx_numbers', Database::INDEX_KEY, ['tv_show', 'numbers'], [], []); // [700, 255]
                 $this->fail('Failed to throw exception');
-            } catch(Throwable $e) {
+            } catch (Throwable $e) {
                 $this->assertEquals('Index length is longer than the maximum: 768', $e->getMessage());
             }
         }
@@ -2718,7 +2718,7 @@ public function testArrayAttribute(): void
         try {
             $database->createIndex($collection, 'indx4', Database::INDEX_KEY, ['age', 'names'], [10, 255], []);
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('Cannot set a length on "integer" attributes', $e->getMessage());
         }
 
@@ -2731,7 +2731,7 @@ public function testArrayAttribute(): void
                     Query::equal('names', ['Joe']),
                 ]);
                 $this->fail('Failed to throw exception');
-            } catch(Throwable $e) {
+            } catch (Throwable $e) {
                 $this->assertEquals('Invalid query: Cannot query equal on attribute "names" because it is an array.', $e->getMessage());
             }
 
@@ -2740,7 +2740,7 @@ public function testArrayAttribute(): void
                     Query::contains('age', [10])
                 ]);
                 $this->fail('Failed to throw exception');
-            } catch(Throwable $e) {
+            } catch (Throwable $e) {
                 $this->assertEquals('Invalid query: Cannot query contains on attribute "age" because it is not an array or string.', $e->getMessage());
             }
 
@@ -3176,7 +3176,7 @@ public function testFindContains(): void
                 Query::contains('price', [10.5]),
             ]);
             $this->fail('Failed to throw exception');
-        } catch(Throwable $e) {
+        } catch (Throwable $e) {
             $this->assertEquals('Invalid query: Cannot query contains on attribute "price" because it is not an array or string.', $e->getMessage());
             $this->assertTrue($e instanceof DatabaseException);
         }
@@ -4083,7 +4083,7 @@ public function testOrSingleQuery(): void
                 ])
             ]);
             $this->fail('Failed to throw exception');
-        } catch(Exception $e) {
+        } catch (Exception $e) {
             $this->assertEquals('Invalid query: Or queries require at least two queries', $e->getMessage());
         }
     }
@@ -4143,7 +4143,7 @@ public function testAndSingleQuery(): void
                 ])
             ]);
             $this->fail('Failed to throw exception');
-        } catch(Exception $e) {
+        } catch (Exception $e) {
             $this->assertEquals('Invalid query: And queries require at least two queries', $e->getMessage());
         }
     }
@@ -4930,7 +4930,7 @@ public function testStructureValidationAfterRelationsAttribute(): void
                 'name' => 'Frozen', // Unknown attribute 'name' after relation attribute
             ]));
             $this->fail('Failed to throw exception');
-        } catch(Exception $e) {
+        } catch (Exception $e) {
             $this->assertInstanceOf(StructureException::class, $e);
         }
     }
@@ -5005,7 +5005,7 @@ public function testNoChangeUpdateDocumentWithRelationWithoutPermission(): void
         try {
             static::getDatabase()->updateDocument('level1', $level1->getId(), $level1->setAttribute('name', 'haha'));
             $this->fail('Failed to throw exception');
-        } catch(Exception $e) {
+        } catch (Exception $e) {
             $this->assertInstanceOf(AuthorizationException::class, $e);
         }
         $level1->setAttribute('name', 'Level 1');
diff --git a/tests/unit/Validator/DateTimeTest.php b/tests/unit/Validator/DateTimeTest.php
index b9157e8b8..d8ababb6f 100644
--- a/tests/unit/Validator/DateTimeTest.php
+++ b/tests/unit/Validator/DateTimeTest.php
@@ -128,7 +128,7 @@ public function testOffset(): void
         $threwException = false;
         try {
             $dateValidator = new DatetimeValidator(offset: -60);
-        } catch(\Exception $e) {
+        } catch (\Exception $e) {
             $threwException = true;
         }
         $this->assertTrue($threwException);

From b3cf76764ccb22a22e6fe680b777f0a145747743 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 13 Sep 2024 23:40:41 +1200
Subject: [PATCH 072/100] Add rollbacks on bad state

---
 src/Database/Adapter/SQL.php | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index d7bce4f3a..1e1bfaebc 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -34,6 +34,10 @@ public function startTransaction(): bool
     {
         try {
             if ($this->inTransaction === 0) {
+                if ($this->getPDO()->inTransaction()) {
+                    $this->getPDO()->rollBack();
+                }
+
                 $result = $this->getPDO()->beginTransaction();
             } else {
                 $result = true;
@@ -66,12 +70,18 @@ public function commitTransaction(): bool
         try {
             $result = $this->getPDO()->commit();
         } catch (PDOException $e) {
+            if ($this->getPDO()->inTransaction()) {
+                $this->getPDO()->rollBack();
+            }
             throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e);
         } finally {
             $this->inTransaction--;
         }
 
         if (!$result) {
+            if ($this->getPDO()->inTransaction()) {
+                $this->getPDO()->rollBack();
+            }
             throw new DatabaseException('Failed to commit transaction');
         }
 

From 238883ab8a80cb107c5dc75092178df6a077dae9 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 13:34:20 +1300
Subject: [PATCH 073/100] Exception simplification

---
 src/Database/Adapter/MariaDB.php  | 48 ++++++-------------------------
 src/Database/Adapter/MySQL.php    | 23 +--------------
 src/Database/Adapter/Postgres.php | 11 ++-----
 3 files changed, 11 insertions(+), 71 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index bef86ebdc..ca9f0eba9 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -948,16 +948,8 @@ public function createDocument(string $collection, Document $document): Document
             if (isset($stmtPermissions)) {
                 $stmtPermissions->execute();
             }
-        } catch (\Throwable $e) {
-            if ($e instanceof PDOException) {
-                switch ($e->getCode()) {
-                    case 1062:
-                    case 23000:
-                        throw new DuplicateException('Duplicated document: ' . $e->getMessage(), previous: $e);
-                }
-            }
-
-            throw $e;
+        } catch (PDOException $e) {
+            throw $this->processException($e);
         }
 
         return $document;
@@ -1079,16 +1071,8 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $stmtPermissions?->execute();
                 }
             }
-        } catch (\Throwable $e) {
-            if ($e instanceof PDOException) {
-                switch ($e->getCode()) {
-                    case 1062:
-                    case 23000:
-                        throw new DuplicateException('Duplicated document: ' . $e->getMessage(), previous: $e);
-                }
-            }
-
-            throw $e;
+        } catch (PDOException $e) {
+            throw $this->processException($e);
         }
 
         foreach ($documents as $document) {
@@ -1337,16 +1321,8 @@ public function updateDocument(string $collection, string $id, Document $documen
                 $stmtAddPermissions->execute();
             }
 
-        } catch (\Throwable $e) {
-            if ($e instanceof PDOException) {
-                switch ($e->getCode()) {
-                    case 1062:
-                    case 23000:
-                        throw new DuplicateException('Duplicated document: ' . $e->getMessage(), previous: $e);
-                }
-            }
-
-            throw $e;
+        } catch (PDOException $e) {
+            throw $this->processException($e);
         }
 
         return $document;
@@ -1594,16 +1570,8 @@ public function updateDocuments(string $collection, array $documents, int $batch
                     $stmtAddPermissions->execute();
                 }
             }
-        } catch (\Throwable $e) {
-            if ($e instanceof PDOException) {
-                switch ($e->getCode()) {
-                    case 1062:
-                    case 23000:
-                        throw new DuplicateException('Duplicated document: ' . $e->getMessage(), previous: $e);
-                }
-            }
-
-            throw $e;
+        } catch (PDOException $e) {
+            $this->processException($e);
         }
 
         return $documents;
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index c0ac85f88..504dec2d1 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -91,27 +91,6 @@ protected function processException(PDOException $e): \Exception
             return new TimeoutException($e->getMessage(), $e->getCode(), $e);
         }
 
-        // Duplicate table
-        if ($e->getCode() === '42S01' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1050) {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        }
-
-        // Duplicate column
-        if ($e->getCode() === '42S21' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1060) {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        }
-
-        // Duplicate index
-        if ($e->getCode() === '42000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1061) {
-            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
-        }
-
-        // Data is too big for column resize
-        if (($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) ||
-            ($e->getCode() === '01000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1265)) {
-            return new TruncateException('Resize would result in data truncation', $e->getCode(), $e);
-        }
-
-        return $e;
+        return parent::processException($e);
     }
 }
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index c7ae88be9..de42e10a8 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -1260,6 +1260,7 @@ public function updateDocument(string $collection, string $id, Document $documen
      * @return array<Document>
      * @throws DatabaseException
      * @throws DuplicateException
+     * @throws Exception
      */
     public function updateDocuments(string $collection, array $documents, int $batchSize = Database::INSERT_BATCH_SIZE): array
     {
@@ -1491,15 +1492,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
             return $documents;
         } catch (PDOException $e) {
-
-            // Must be a switch for loose match
-            switch ($e->getCode()) {
-                case 1062:
-                case 23505:
-                    throw new DuplicateException('Duplicated document: ' . $e->getMessage());
-                default:
-                    throw $e;
-            }
+            throw $this->processException($e);
         }
     }
 

From 9105080161087ea5f14a88405315ad7b124d2677 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 13:42:23 +1300
Subject: [PATCH 074/100] Remove redundant get document

---
 dev/xdebug.ini            | 2 +-
 src/Database/Database.php | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/dev/xdebug.ini b/dev/xdebug.ini
index 34ec0e4b6..9fdfa2d76 100644
--- a/dev/xdebug.ini
+++ b/dev/xdebug.ini
@@ -3,10 +3,10 @@ zend_extension = xdebug.so
 [xdebug]
 xdebug.mode = develop,debug,profile
 xdebug.start_with_request = yes
+xdebug.use_compression=false
 xdebug.client_host=host.docker.internal
 xdebug.client_port = 9003
 xdebug.log = /tmp/xdebug.log
-xdebug.use_compression=false
 
 xdebug.var_display_max_depth = 10
 xdebug.var_display_max_children = 256
diff --git a/src/Database/Database.php b/src/Database/Database.php
index 0b631d409..4cebb9a91 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -4516,12 +4516,9 @@ public function decreaseDocumentAttribute(string $collection, string $id, string
      * @throws ConflictException
      * @throws DatabaseException
      * @throws RestrictedException
-     * @throws StructureException
      */
     public function deleteDocument(string $collection, string $id): bool
     {
-        $document = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id)));
-
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
         $deleted = $this->withTransaction(function () use ($collection, $id, &$document) {

From fde34787ab35383a33ded502a3b5327766cc0be2 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 13:46:48 +1300
Subject: [PATCH 075/100] Lint/check fixes

---
 composer.json                     | 2 +-
 src/Database/Adapter/MySQL.php    | 2 --
 src/Database/Adapter/Postgres.php | 6 +-----
 3 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/composer.json b/composer.json
index d0097c5d5..a99e03468 100755
--- a/composer.json
+++ b/composer.json
@@ -29,7 +29,7 @@
         ],
         "lint": "./vendor/bin/pint --test",
         "format": "./vendor/bin/pint",
-        "check": "./vendor/bin/phpstan analyse --level 7 src tests --memory-limit 512M",
+        "check": "./vendor/bin/phpstan analyse --level 7 src tests --memory-limit 2G",
         "coverage": "./vendor/bin/coverage-check ./tmp/clover.xml 90"
     },
     "require": {
diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php
index 504dec2d1..7564d5a51 100644
--- a/src/Database/Adapter/MySQL.php
+++ b/src/Database/Adapter/MySQL.php
@@ -5,9 +5,7 @@
 use PDOException;
 use Utopia\Database\Database;
 use Utopia\Database\Exception as DatabaseException;
-use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
-use Utopia\Database\Exception\Truncate as TruncateException;
 
 class MySQL extends MariaDB
 {
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index de42e10a8..f6ae3a6fc 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -904,9 +904,8 @@ public function createDocument(string $collection, Document $document): Document
             if (isset($stmtPermissions)) {
                 $stmtPermissions->execute();
             }
-        } catch (Throwable $e) {
+        } catch (PDOException $e) {
             throw $this->processException($e);
-
         }
 
         return $document;
@@ -1007,9 +1006,6 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $stmtPermissions?->execute();
                 }
             }
-
-            return $documents;
-
         } catch (PDOException $e) {
             throw $this->processException($e);
         }

From 790b870631a8061a335619d2ccb163ccc05235cf Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 13:54:06 +1300
Subject: [PATCH 076/100] Fix merge

---
 src/Database/Adapter/Postgres.php | 1 -
 src/Database/Adapter/SQL.php      | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index f6ae3a6fc..29a6aee64 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -5,7 +5,6 @@
 use Exception;
 use PDO;
 use PDOException;
-use Throwable;
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Exception as DatabaseException;
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index 80a3152f2..1e1bfaebc 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -211,10 +211,6 @@ public function getDocument(string $collection, string $id, array $queries = [],
             $sql .= " {$forUpdate}";
         }
 
-        if ($this->getSupportForUpdateLock()) {
-            $sql .= " {$forUpdate}";
-        }
-
         $stmt = $this->getPDO()->prepare($sql);
 
         $stmt->bindValue(':_uid', $id);

From 68f5bf6fc8838ed4a589f76842c6dbc4208adf4e Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 14:56:02 +1300
Subject: [PATCH 077/100] Fix tests

---
 src/Database/Adapter/MariaDB.php  |  5 +++++
 src/Database/Adapter/Postgres.php |  5 +++++
 src/Database/Adapter/SQLite.php   |  2 +-
 tests/e2e/Adapter/Base.php        | 17 +++++++++++++----
 4 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index ca9f0eba9..9dba28900 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -2306,6 +2306,11 @@ protected function processException(PDOException $e): \Exception
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
+        // Duplicate row
+        if ($e->getCode() === '23000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1062) {
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        }
+
         // Data is too big for column resize
         if (($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1406) ||
             ($e->getCode() === '01000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1265)) {
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index 29a6aee64..15f919dd9 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -2272,6 +2272,11 @@ protected function processException(PDOException $e): \Exception
             return new DuplicateException($e->getMessage(), $e->getCode(), $e);
         }
 
+        // Duplicate row
+        if ($e->getCode() === '23505' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
+            return new DuplicateException($e->getMessage(), $e->getCode(), $e);
+        }
+
         // Data is too big for column resize
         if ($e->getCode() === '22001' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 7) {
             return new TruncateException('Resize would result in data truncation', $e->getCode(), $e);
diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index 1eac1d516..7db1760ba 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -263,7 +263,7 @@ public function deleteCollection(string $id): bool
     {
         $id = $this->filter($id);
 
-        $sql = "DROP TABLE IF EXISTS `{$this->getSQLTable($id)}`";
+        $sql = "DROP TABLE IF EXISTS {$this->getSQLTable($id)}";
         $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql);
 
         $this->getPDO()
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 9d8559b1d..e04213933 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -15612,7 +15612,7 @@ public function testSharedTables(): void
         $database->setDatabase($this->testDatabase);
     }
 
-    public function testSharedTablesDuplicatesDontThrow(): void
+    public function testSharedTablesDuplicates(): void
     {
         $database = static::getDatabase();
 
@@ -15629,7 +15629,7 @@ public function testSharedTablesDuplicatesDontThrow(): void
             ->setDatabase('sharedTables')
             ->setNamespace('')
             ->setSharedTables(true)
-            ->setTenant(1)
+            ->setTenant(null)
             ->create();
 
         // Create collection
@@ -15645,8 +15645,17 @@ public function testSharedTablesDuplicatesDontThrow(): void
             // Ignore
         }
 
-        $database->createAttribute('duplicates', 'name', Database::VAR_STRING, 10, false);
-        $database->createIndex('duplicates', 'nameIndex', Database::INDEX_KEY, ['name']);
+        try {
+            $database->createAttribute('duplicates', 'name', Database::VAR_STRING, 10, false);
+        } catch (DuplicateException) {
+            // Ignore
+        }
+
+        try {
+            $database->createIndex('duplicates', 'nameIndex', Database::INDEX_KEY, ['name']);
+        } catch (DuplicateException) {
+            // Ignore
+        }
 
         $collection = $database->getCollection('duplicates');
         $this->assertEquals(1, \count($collection->getAttribute('attributes')));

From 74d7f8db9a4670c470681c2d4f6ce70e61bdf7cd Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 16:53:56 +1300
Subject: [PATCH 078/100] Fix cache not flushed on db delete

---
 src/Database/Database.php | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 4cebb9a91..00bf9bd9c 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -1076,6 +1076,8 @@ public function delete(?string $database = null): bool
             'deleted' => $deleted
         ]);
 
+        $this->cache->flush();
+
         return $deleted;
     }
 

From e06001b5785ced2f2994f885680c958d0a91b7a1 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 16:54:15 +1300
Subject: [PATCH 079/100] Fix upgrade read on source/dest mirror

---
 src/Database/Mirror.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index 6ff3860a7..bb0b44ca8 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -901,7 +901,7 @@ protected function getUpgradeStatus(string $collection): ?Document
 
         return Authorization::skip(function () use ($collection) {
             try {
-                return $this->getDocument('upgrades', $collection);
+                return $this->source->getDocument('upgrades', $collection);
             } catch (\Throwable) {
                 return;
             }

From ec47a5e31126aba2c79d8d2e66aa99b6b309da52 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Mon, 14 Oct 2024 18:01:22 +1300
Subject: [PATCH 080/100] Fix max allowed packet on mirror

---
 docker-compose.yml               | 1 +
 src/Database/Adapter/MariaDB.php | 2 +-
 src/Database/Database.php        | 2 +-
 tests/e2e/Adapter/Base.php       | 3 +--
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 82076ebfe..76614e18b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -50,6 +50,7 @@ services:
   mariadb-mirror:
     image: mariadb:10.11
     container_name: utopia-mariadb-mirror
+    command: mariadbd --max_allowed_packet=1G
     networks:
       - database
     ports:
diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index 9dba28900..7465db0ec 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -1816,7 +1816,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25,
 
         $sql = "
             SELECT {$this->getAttributeProjection($selections, 'table_main')}
-            FROM {$this->getSQLTable($name)} as table_main
+            FROM {$this->getSQLTable($name)} AS table_main
             {$sqlWhere}
             {$sqlOrder}
             {$sqlLimit};
diff --git a/src/Database/Database.php b/src/Database/Database.php
index 00bf9bd9c..080dea1f7 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -1722,7 +1722,7 @@ public function updateAttributeDefault(string $collection, string $id, mixed $de
 
     /**
      * Update Attribute. This method is for updating data that causes underlying structure to change. Check out other updateAttribute methods if you are looking for metadata adjustments.
-     * To update attribute key (ID), use renameAttribute instead.
+     *
      * @param string $collection
      * @param string $id
      * @param string|null $type
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index e04213933..2ddeb9107 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -6123,8 +6123,7 @@ public function testUpdateAttributeSize(): void
         $document = $this->updateStringAttributeSize(65536, $document);
 
         // 65536-16777216 to PHP_INT_MAX or adapter limit
-        $maxStringSize = 16777217;
-        $document = $this->updateStringAttributeSize($maxStringSize, $document);
+        $document = $this->updateStringAttributeSize(16777217, $document);
 
         // Test going down in size with data that is too big (Expect Failure)
         try {

From 58c7c6843a3010e801d0c48a30c96ec0b78409ec Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 14:00:52 +1300
Subject: [PATCH 081/100] Improve delegation

---
 src/Database/Mirror.php | 48 ++++++++++++++++++++---------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index bb0b44ca8..f607ee3a4 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -104,35 +104,35 @@ protected function delegate(string $method, array $args = []): mixed
 
     public function setDatabase(string $name): static
     {
-        $this->delegate('setDatabase', [$name]);
+        $this->delegate(__FUNCTION__, \func_get_args());
 
         return $this;
     }
 
     public function setNamespace(string $namespace): static
     {
-        $this->delegate('setNamespace', [$namespace]);
+        $this->delegate(__FUNCTION__, \func_get_args());
 
         return $this;
     }
 
     public function setSharedTables(bool $sharedTables): static
     {
-        $this->delegate('setSharedTables', [$sharedTables]);
+        $this->delegate(__FUNCTION__, \func_get_args());
 
         return $this;
     }
 
     public function setTenant(?int $tenant): static
     {
-        $this->delegate('setTenant', [$tenant]);
+        $this->delegate(__FUNCTION__, \func_get_args());
 
         return $this;
     }
 
     public function setPreserveDates(bool $preserve): static
     {
-        $this->delegate('setPreserveDates', [$preserve]);
+        $this->delegate(__FUNCTION__, \func_get_args());
 
         $this->preserveDates = $preserve;
 
@@ -141,7 +141,7 @@ public function setPreserveDates(bool $preserve): static
 
     public function enableValidation(): static
     {
-        $this->delegate('enableValidation');
+        $this->delegate(__FUNCTION__);
 
         $this->validate = true;
 
@@ -150,7 +150,7 @@ public function enableValidation(): static
 
     public function disableValidation(): static
     {
-        $this->delegate('disableValidation');
+        $this->delegate(__FUNCTION__);
 
         $this->validate = false;
 
@@ -176,22 +176,22 @@ public function silent(callable $callback, array $listeners = null): mixed
 
     public function withRequestTimestamp(?\DateTime $requestTimestamp, callable $callback): mixed
     {
-        return $this->delegate('withRequestTimestamp', [$requestTimestamp, $callback]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function exists(?string $database = null, ?string $collection = null): bool
     {
-        return $this->delegate('exists', [$database, $collection]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function create(?string $database = null): bool
     {
-        return $this->delegate('create', [$database]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function delete(?string $database = null): bool
     {
-        return $this->delegate('delete', [$database]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function createCollection(string $id, array $attributes = [], array $indexes = [], array $permissions = null, bool $documentSecurity = true): Document
@@ -760,32 +760,32 @@ public function deleteDocument(string $collection, string $id): bool
 
     public function updateAttributeRequired(string $collection, string $id, bool $required): Document
     {
-        return $this->delegate('updateAttributeRequired', [$collection, $id, $required]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function updateAttributeFormat(string $collection, string $id, string $format): Document
     {
-        return $this->delegate('updateAttributeFormat', [$collection, $id, $format]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function updateAttributeFormatOptions(string $collection, string $id, array $formatOptions): Document
     {
-        return $this->delegate('updateAttributeFormatOptions', [$collection, $id, $formatOptions]);
+        return $this->delegate(__FUNCTION__, [$collection, $id, $formatOptions]);
     }
 
     public function updateAttributeFilters(string $collection, string $id, array $filters): Document
     {
-        return $this->delegate('updateAttributeFilters', [$collection, $id, $filters]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function updateAttributeDefault(string $collection, string $id, mixed $default = null): Document
     {
-        return $this->delegate('updateAttributeDefault', [$collection, $id, $default]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function renameAttribute(string $collection, string $old, string $new): bool
     {
-        return $this->delegate('renameAttribute', [$collection, $old, $new]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function createRelationship(
@@ -797,7 +797,7 @@ public function createRelationship(
         ?string $twoWayKey = null,
         string $onDelete = Database::RELATION_MUTATE_RESTRICT
     ): bool {
-        return $this->delegate('createRelationship', [$collection, $relatedCollection, $type, $twoWay, $id, $twoWayKey, $onDelete]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function updateRelationship(
@@ -808,28 +808,28 @@ public function updateRelationship(
         ?bool $twoWay = null,
         ?string $onDelete = null
     ): bool {
-        return $this->delegate('updateRelationship', [$collection, $id, $newKey, $newTwoWayKey, $twoWay, $onDelete]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function deleteRelationship(string $collection, string $id): bool
     {
-        return $this->delegate('deleteRelationship', [$collection, $id]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
 
     public function renameIndex(string $collection, string $old, string $new): bool
     {
-        return $this->delegate('renameIndex', [$collection, $old, $new]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $max = null): bool
     {
-        return $this->delegate('increaseDocumentAttribute', [$collection, $id, $attribute, $value, $max]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     public function decreaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $min = null): bool
     {
-        return $this->delegate('decreaseDocumentAttribute', [$collection, $id, $attribute, $value, $min]);
+        return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
     /**
@@ -903,7 +903,7 @@ protected function getUpgradeStatus(string $collection): ?Document
             try {
                 return $this->source->getDocument('upgrades', $collection);
             } catch (\Throwable) {
-                return;
+                return null;
             }
         });
     }

From 6639898d2aa6e58fd2c78f6a0bc3ff2bf799e4b4 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 14:01:14 +1300
Subject: [PATCH 082/100] Fix spelling

---
 Dockerfile                 |  2 +-
 tests/e2e/Adapter/Base.php | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 87fca91e3..f7985eba1 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,7 @@ RUN composer install \
     --no-scripts \
     --prefer-dist
     
-FROM php:8.3.10-cli-alpine3.20 as compile
+FROM php:8.3.10-cli-alpine3.20 AS compile
 
 ENV PHP_REDIS_VERSION="6.0.2" \
     PHP_SWOOLE_VERSION="v5.1.3" \
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 2ddeb9107..d569458ee 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -6446,11 +6446,11 @@ public function testNoInvalidKeysWithRelationships(): void
         }
         static::getDatabase()->createCollection('species');
         static::getDatabase()->createCollection('creatures');
-        static::getDatabase()->createCollection('characterstics');
+        static::getDatabase()->createCollection('characteristics');
 
         static::getDatabase()->createAttribute('species', 'name', Database::VAR_STRING, 255, true);
         static::getDatabase()->createAttribute('creatures', 'name', Database::VAR_STRING, 255, true);
-        static::getDatabase()->createAttribute('characterstics', 'name', Database::VAR_STRING, 255, true);
+        static::getDatabase()->createAttribute('characteristics', 'name', Database::VAR_STRING, 255, true);
 
         static::getDatabase()->createRelationship(
             collection: 'species',
@@ -6462,10 +6462,10 @@ public function testNoInvalidKeysWithRelationships(): void
         );
         static::getDatabase()->createRelationship(
             collection: 'creatures',
-            relatedCollection: 'characterstics',
+            relatedCollection: 'characteristics',
             type: Database::RELATION_ONE_TO_ONE,
             twoWay: true,
-            id: 'characterstic',
+            id: 'characteristic',
             twoWayKey:'creature'
         );
 
@@ -6481,7 +6481,7 @@ public function testNoInvalidKeysWithRelationships(): void
                     Permission::read(Role::any()),
                 ],
                 'name' => 'Dog',
-                'characterstic' => [
+                'characteristic' => [
                     '$id' => ID::custom('1'),
                     '$permissions' => [
                         Permission::read(Role::any()),
@@ -6497,10 +6497,10 @@ public function testNoInvalidKeysWithRelationships(): void
             'creature' => [
                 '$id' => ID::custom('1'),
                 '$collection' => 'creatures',
-                'characterstic' => [
+                'characteristic' => [
                     '$id' => ID::custom('1'),
                     'name' => 'active',
-                    '$collection' => 'characterstics',
+                    '$collection' => 'characteristics',
                 ]
             ]
         ]));

From a9de51b4817c26eaf7d9da8c494ca4dc07292c75 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 14:04:08 +1300
Subject: [PATCH 083/100] Add redis mirror

---
 docker-compose.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 76614e18b..70df221eb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -102,12 +102,20 @@ services:
       - SYS_NICE
 
   redis:
-    image: redis:6.0-alpine
+    image: redis:7.4.1-alpine3.20
     container_name: utopia-redis
     ports:
       - "8706:6379"
     networks:
       - database
 
+  redis-mirror:
+    image: redis:7.4.1-alpine3.20
+    container_name: utopia-redis
+    ports:
+      - "8707:6379"
+    networks:
+      - database
+
 networks:
   database:

From 30e470fdebb1f2f027ed3816493abd78457f8ed5 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 14:16:44 +1300
Subject: [PATCH 084/100] Use mirror cache for mirror db

---
 docker-compose.yml               | 31 +++++++++++++++++++++----------
 tests/e2e/Adapter/Base.php       |  7 ++-----
 tests/e2e/Adapter/MirrorTest.php |  6 +++++-
 3 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 70df221eb..1672fbca1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,17 +21,28 @@ services:
     container_name: utopia-adminer
     restart: always
     ports:
-      - "8760:8080"
+      - "8700:8080"
     networks:
       - database
 
   postgres:
-    image: postgres:13
+    image: postgres:16.4
     container_name: utopia-postgres
     networks:
       - database
     ports:
-      - "8700:5432"
+      - "8701:5432"
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: password
+
+  postgres-mirror:
+    image: postgres:16.4
+    container_name: utopia-postgres-mirror
+    networks:
+      - database
+    ports:
+      - "8702:5432"
     environment:
       POSTGRES_USER: root
       POSTGRES_PASSWORD: password
@@ -43,7 +54,7 @@ services:
     networks:
       - database
     ports:
-      - "8701:3306"
+      - "8703:3306"
     environment:
       - MYSQL_ROOT_PASSWORD=password
 
@@ -64,7 +75,7 @@ services:
     networks:
       - database
     ports:
-      - "8702:27017"
+      - "8705:27017"
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: example
@@ -75,7 +86,7 @@ services:
     networks:
       - database
     ports:
-      - "8703:3307"
+      - "8706:3307"
     environment:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_DATABASE: default
@@ -91,7 +102,7 @@ services:
     networks:
       - database
     ports:
-      - "8705:3307"
+      - "8707:3307"
     environment:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_DATABASE: default
@@ -105,15 +116,15 @@ services:
     image: redis:7.4.1-alpine3.20
     container_name: utopia-redis
     ports:
-      - "8706:6379"
+      - "8708:6379"
     networks:
       - database
 
   redis-mirror:
     image: redis:7.4.1-alpine3.20
-    container_name: utopia-redis
+    container_name: utopia-redis-mirror
     ports:
-      - "8707:6379"
+      - "8709:6379"
     networks:
       - database
 
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index d569458ee..26bba07f1 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -6354,14 +6354,9 @@ public function testKeywords(): void
             $this->assertCount(1, $documents);
             $this->assertEquals('reservedKeyDocument', $documents[0]->getId());
 
-
             $collection = $database->deleteCollection($collectionName);
             $this->assertTrue($collection);
-
-            // TODO: updateAttribute name tests
         }
-
-        // TODO: Index name tests
     }
 
     public function testWritePermissions(): void
@@ -6504,7 +6499,9 @@ public function testNoInvalidKeysWithRelationships(): void
                 ]
             ]
         ]));
+
         $updatedSpecies = static::getDatabase()->getDocument('species', $species->getId());
+
         $this->assertEquals($species, $updatedSpecies);
     }
 
diff --git a/tests/e2e/Adapter/MirrorTest.php b/tests/e2e/Adapter/MirrorTest.php
index 8a0bef837..efcfb04c8 100644
--- a/tests/e2e/Adapter/MirrorTest.php
+++ b/tests/e2e/Adapter/MirrorTest.php
@@ -56,8 +56,12 @@ protected static function getDatabase(bool $fresh = false): Mirror
         $mirrorPass = 'password';
 
         $mirrorPdo = new PDO("mysql:host={$mirrorHost};port={$mirrorPort};charset=utf8mb4", $mirrorUser, $mirrorPass, MariaDB::getPDOAttributes());
+        $mirrorRedis = new Redis();
+        $mirrorRedis->connect('redis-mirror');
+        $mirrorRedis->flushAll();
+        $mirrorCache = new Cache(new RedisAdapter($mirrorRedis));
 
-        self::$destination = new Database(new MariaDB($mirrorPdo), $cache);
+        self::$destination = new Database(new MariaDB($mirrorPdo), $mirrorCache);
 
         $database = new Mirror(self::$source, self::$destination);
 

From 781b421fca340c0b4628407eb66c502061172fc2 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 14:23:10 +1300
Subject: [PATCH 085/100] Lint

---
 src/Database/Mirror.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php
index f607ee3a4..2f9f1d186 100644
--- a/src/Database/Mirror.php
+++ b/src/Database/Mirror.php
@@ -903,7 +903,7 @@ protected function getUpgradeStatus(string $collection): ?Document
             try {
                 return $this->source->getDocument('upgrades', $collection);
             } catch (\Throwable) {
-                return null;
+                return;
             }
         });
     }

From 651cb4ffbcb743676a784b0c118f38f3bafff2e5 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 15:30:09 +1300
Subject: [PATCH 086/100] Fix SQLite tests

---
 src/Database/Adapter/SQLite.php | 22 +++++++++++-----------
 tests/e2e/Adapter/Base.php      |  2 +-
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php
index 7db1760ba..cb75411ac 100644
--- a/src/Database/Adapter/SQLite.php
+++ b/src/Database/Adapter/SQLite.php
@@ -280,6 +280,17 @@ public function deleteCollection(string $id): bool
         return true;
     }
 
+    /**
+     * Analyze a collection updating it's metadata on the database engine
+     *
+     * @param string $collection
+     * @return bool
+     */
+    public function analyzeCollection(string $collection): bool
+    {
+        return false;
+    }
+
     /**
      * Update Attribute
      *
@@ -1393,15 +1404,4 @@ protected function processException(PDOException $e): \Exception
 
         return $e;
     }
-
-    /**
-     * Analyze a collection updating it's metadata on the database engine
-     *
-     * @param string $collection
-     * @return bool
-     */
-    public function analyzeCollection(string $collection): bool
-    {
-        return false;
-    }
 }
diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php
index 26bba07f1..45eb64859 100644
--- a/tests/e2e/Adapter/Base.php
+++ b/tests/e2e/Adapter/Base.php
@@ -15612,7 +15612,7 @@ public function testSharedTablesDuplicates(): void
     {
         $database = static::getDatabase();
 
-        if (!$database->getAdapter()->getSupportForAttributes()) {
+        if (!$database->getAdapter()->getSupportForSchemas()) {
             $this->expectNotToPerformAssertions();
             return;
         }

From b63c999e3a0617779e1f84480267d7d18622b542 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 16:52:13 +1300
Subject: [PATCH 087/100] Fix order of operations issues

---
 tests/e2e/Adapter/MirrorTest.php | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tests/e2e/Adapter/MirrorTest.php b/tests/e2e/Adapter/MirrorTest.php
index efcfb04c8..ee34386f9 100644
--- a/tests/e2e/Adapter/MirrorTest.php
+++ b/tests/e2e/Adapter/MirrorTest.php
@@ -65,6 +65,26 @@ protected static function getDatabase(bool $fresh = false): Mirror
 
         $database = new Mirror(self::$source, self::$destination);
 
+        // Handle cases where the source and destination databases are not in sync because of previous tests
+        if ($database->getSource()->exists('schema1')) {
+            $database->getSource()->setDatabase('schema1')->delete();
+        }
+        if ($database->getDestination()->exists('schema1')) {
+            $database->getDestination()->setDatabase('schema1')->delete();
+        }
+        if ($database->getSource()->exists('schema2')) {
+            $database->getSource()->setDatabase('schema2')->delete();
+        }
+        if ($database->getDestination()->exists('schema2')) {
+            $database->getDestination()->setDatabase('schema2')->delete();
+        }
+        if ($database->getSource()->exists('sharedTables')) {
+            $database->getSource()->setDatabase('sharedTables')->delete();
+        }
+        if ($database->getDestination()->exists('sharedTables')) {
+            $database->getDestination()->setDatabase('sharedTables')->delete();
+        }
+
         $database
             ->setDatabase('utopiaTests')
             ->setNamespace(static::$namespace = 'myapp_' . uniqid());

From 8bb5793b6e252e2ec840b7056d74e65767c287e8 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 16:52:20 +1300
Subject: [PATCH 088/100] Fix mongo tests

---
 src/Database/Adapter/Mongo.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php
index b15aef596..19b33024c 100644
--- a/src/Database/Adapter/Mongo.php
+++ b/src/Database/Adapter/Mongo.php
@@ -1593,7 +1593,7 @@ public function getLimitForIndexes(): int
      */
     public function getSupportForSchemas(): bool
     {
-        return true;
+        return false;
     }
 
     /**

From 78f0e9d08b5f414b303a6116d2e5cb379b312e97 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 18:41:32 +1300
Subject: [PATCH 089/100] Bail out of commit if no active transaction

---
 src/Database/Adapter/SQL.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index 1e1bfaebc..34dcea386 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -67,6 +67,11 @@ public function commitTransaction(): bool
             return true;
         }
 
+        if (!$this->getPDO()->inTransaction()) {
+            $this->inTransaction = 0;
+            return false;
+        }
+
         try {
             $result = $this->getPDO()->commit();
         } catch (PDOException $e) {

From efd0f4e8fb5d158486aec0e6fea9561facf0daad Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 15 Oct 2024 18:50:09 +1300
Subject: [PATCH 090/100] Fix stan

---
 src/Database/Adapter/SQL.php | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index 34dcea386..ae2dbea22 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -68,6 +68,7 @@ public function commitTransaction(): bool
         }
 
         if (!$this->getPDO()->inTransaction()) {
+            // Implicit commit occurred
             $this->inTransaction = 0;
             return false;
         }
@@ -75,18 +76,12 @@ public function commitTransaction(): bool
         try {
             $result = $this->getPDO()->commit();
         } catch (PDOException $e) {
-            if ($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
             throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e);
         } finally {
             $this->inTransaction--;
         }
 
         if (!$result) {
-            if ($this->getPDO()->inTransaction()) {
-                $this->getPDO()->rollBack();
-            }
             throw new DatabaseException('Failed to commit transaction');
         }
 

From 5b9462f8abc12c2c7ab2dac99aa6f68aa2e27365 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 22 Oct 2024 21:05:24 +1300
Subject: [PATCH 091/100] Add log dumps

---
 src/Database/Adapter/SQL.php | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index ae2dbea22..6cd92e155 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -35,10 +35,12 @@ public function startTransaction(): bool
         try {
             if ($this->inTransaction === 0) {
                 if ($this->getPDO()->inTransaction()) {
+                    \var_dump('Rolling back active transaction');
                     $this->getPDO()->rollBack();
                 }
 
                 $result = $this->getPDO()->beginTransaction();
+                \var_dump('Started transaction');
             } else {
                 $result = true;
             }
@@ -48,10 +50,11 @@ public function startTransaction(): bool
 
         if (!$result) {
             throw new DatabaseException('Failed to start transaction');
+        } else {
+            \var_dump('Incrementing transaction count');
+            $this->inTransaction++;
         }
 
-        $this->inTransaction++;
-
         return $result;
     }
 
@@ -61,23 +64,28 @@ public function startTransaction(): bool
     public function commitTransaction(): bool
     {
         if ($this->inTransaction === 0) {
+            \var_dump('No transaction to commit');
             return false;
         } elseif ($this->inTransaction > 1) {
+            \var_dump('Decrementing transaction count');
             $this->inTransaction--;
             return true;
         }
 
         if (!$this->getPDO()->inTransaction()) {
             // Implicit commit occurred
+            \var_dump('Implicit commit occurred, resetting transaction count');
             $this->inTransaction = 0;
             return false;
         }
 
         try {
             $result = $this->getPDO()->commit();
+            \var_dump('Committed transaction');
         } catch (PDOException $e) {
             throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e);
         } finally {
+            \var_dump('Decrementing transaction count');
             $this->inTransaction--;
         }
 
@@ -94,14 +102,17 @@ public function commitTransaction(): bool
     public function rollbackTransaction(): bool
     {
         if ($this->inTransaction === 0) {
+            \var_dump('No transaction to rollback');
             return false;
         }
 
         try {
             $result = $this->getPDO()->rollBack();
+            \var_dump('Rolled back transaction');
         } catch (PDOException $e) {
             throw new DatabaseException('Failed to rollback transaction: ' . $e->getMessage(), $e->getCode(), $e);
         } finally {
+            \var_dump('Resetting transaction count');
             $this->inTransaction = 0;
         }
 

From 7bd224ae817776f078e7d22f7c8ee124d465e1cd Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Tue, 22 Oct 2024 22:49:03 +1300
Subject: [PATCH 092/100] Debug

---
 src/Database/Adapter.php     | 7 +++++--
 src/Database/Adapter/SQL.php | 7 ++++++-
 src/Database/Database.php    | 5 +++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php
index 218694d4b..a68dd0e2f 100644
--- a/src/Database/Adapter.php
+++ b/src/Database/Adapter.php
@@ -286,13 +286,16 @@ public function inTransaction(): bool
      */
     public function withTransaction(callable $callback): mixed
     {
-        $this->startTransaction();
-
         try {
+            \var_dump('Start');
+            $this->startTransaction();
             $result = $callback();
+            \var_dump('Commit');
             $this->commitTransaction();
+            \var_dump('Return');
             return $result;
         } catch (\Throwable $e) {
+            \var_dump('Rollback');
             $this->rollbackTransaction();
             throw $e;
         }
diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index 6cd92e155..cc899ceaf 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -34,7 +34,12 @@ public function startTransaction(): bool
     {
         try {
             if ($this->inTransaction === 0) {
-                if ($this->getPDO()->inTransaction()) {
+                $pdo = $this->getPDO();
+                if ($pdo::class === 'Swoole\Database\PDOProxy' && $pdo->inTransaction()) {
+                    \var_dump('Getting raw PDO from proxy');
+                    $pdo = $pdo->__getObject();
+                }
+                if ($pdo->inTransaction()) {
                     \var_dump('Rolling back active transaction');
                     $this->getPDO()->rollBack();
                 }
diff --git a/src/Database/Database.php b/src/Database/Database.php
index c542a9b37..1a9a5501a 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3276,6 +3276,7 @@ public function createDocument(string $collection, Document $document): Document
             throw new StructureException($structure->getDescription());
         }
 
+        \var_dump('before transaction createDocument', $collection, $document);
         $document = $this->withTransaction(function () use ($collection, $document) {
             if ($this->resolveRelationships) {
                 $document = $this->silent(fn () => $this->createDocumentRelationships($collection, $document));
@@ -3342,6 +3343,7 @@ public function createDocuments(string $collection, array $documents, int $batch
             $documents[$key] = $document;
         }
 
+        \var_dump('before transaction createDocuments', $collection);
         $documents = $this->withTransaction(function () use ($collection, $documents, $batchSize) {
             return $this->adapter->createDocuments($collection->getId(), $documents, $batchSize);
         });
@@ -3703,6 +3705,7 @@ public function updateDocument(string $collection, string $id, Document $documen
 
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
+        \var_dump('before transaction updateDocument', $collection, $document);
         $document = $this->withTransaction(function () use ($collection, $id, $document) {
             $time = DateTime::now();
             $old = Authorization::skip(fn () => $this->silent(
@@ -3901,6 +3904,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
 
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
+        \var_dump('before transaction updateDocuments', $collection);
         $documents = $this->withTransaction(function () use ($collection, $documents, $batchSize) {
             $time = DateTime::now();
 
@@ -4551,6 +4555,7 @@ public function deleteDocument(string $collection, string $id): bool
     {
         $collection = $this->silent(fn () => $this->getCollection($collection));
 
+        \var_dump('before transaction deleteDocument', $collection, $id);
         $deleted = $this->withTransaction(function () use ($collection, $id, &$document) {
             $document = Authorization::skip(fn () => $this->silent(
                 fn () =>

From df574633d6540984e5ae8efab806e3b493acca5b Mon Sep 17 00:00:00 2001
From: fogelito <fogelshmuel@gmail.com>
Date: Thu, 31 Oct 2024 18:26:25 +0200
Subject: [PATCH 093/100] Question?

---
 src/Database/Adapter/SQL.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index cc899ceaf..cc49313b4 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -91,6 +91,7 @@ public function commitTransaction(): bool
             throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e);
         } finally {
             \var_dump('Decrementing transaction count');
+            // Should we set this to 0 if we commit?
             $this->inTransaction--;
         }
 

From 33576578f9b4014b2a5acb2c2574f85145299f0f Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 6 Nov 2024 16:39:25 +1300
Subject: [PATCH 094/100] Revert "Question?"

This reverts commit df574633d6540984e5ae8efab806e3b493acca5b.
---
 src/Database/Adapter/SQL.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php
index cc49313b4..cc899ceaf 100644
--- a/src/Database/Adapter/SQL.php
+++ b/src/Database/Adapter/SQL.php
@@ -91,7 +91,6 @@ public function commitTransaction(): bool
             throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e);
         } finally {
             \var_dump('Decrementing transaction count');
-            // Should we set this to 0 if we commit?
             $this->inTransaction--;
         }
 

From a23aec26bb3ab4f1e6a0aff137371f5061efeff6 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 6 Nov 2024 20:49:40 +1300
Subject: [PATCH 095/100] Update index order

---
 src/Database/Adapter/MariaDB.php  | 8 ++++----
 src/Database/Adapter/Postgres.php | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php
index e45189713..425360985 100644
--- a/src/Database/Adapter/MariaDB.php
+++ b/src/Database/Adapter/MariaDB.php
@@ -163,10 +163,10 @@ public function createCollection(string $name, array $attributes = [], array $in
         if ($this->sharedTables) {
             $collection .= "
             	_tenant INT(11) UNSIGNED DEFAULT NULL,
-				UNIQUE KEY _uid (_tenant, _uid),
-				KEY _created_at (_tenant, _createdAt),
-				KEY _updated_at (_tenant, _updatedAt),
-				KEY _tenant_id (_tenant, _id)
+				UNIQUE KEY _uid (_uid, _tenant),
+				KEY _created_at (_createdAt, _tenant),
+				KEY _updated_at (_updatedAt, _tenant),
+				KEY _tenant_id (_id, _tenant)
 			";
         } else {
             $collection .= "
diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php
index 2bb044431..817ca3e2a 100644
--- a/src/Database/Adapter/Postgres.php
+++ b/src/Database/Adapter/Postgres.php
@@ -215,9 +215,9 @@ public function createCollection(string $name, array $attributes = [], array $in
         if ($this->sharedTables) {
             $collection .= "
 				CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid), _tenant);
-            	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\");
-            	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\");
-            	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_tenant_id\" ON {$this->getSQLTable($id)} (_tenant, _id);
+            	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\", _tenant);
+            	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\", _tenant);
+            	CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_tenant_id\" ON {$this->getSQLTable($id)} (_id, _tenant);
 			";
         } else {
             $collection .= "

From 6dde7ef87900024b9e141e37f5014e9ca4a7e913 Mon Sep 17 00:00:00 2001
From: shimon <shimon@appwrite.io>
Date: Thu, 7 Nov 2024 12:21:00 +0200
Subject: [PATCH 096/100] composer

---
 composer.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/composer.json b/composer.json
index d0097c5d5..d779556fb 100755
--- a/composer.json
+++ b/composer.json
@@ -37,7 +37,7 @@
         "ext-mbstring": "*",
         "php": ">=8.0",
         "utopia-php/framework": "0.33.*",
-        "utopia-php/cache": "0.10.*",
+        "utopia-php/cache": "0.11.*",
         "utopia-php/mongo": "0.3.*"
     },
     "require-dev": {

From 5af02bf3e3970c1cf0272954f6ee485ad42432e9 Mon Sep 17 00:00:00 2001
From: shimon <shimon@appwrite.io>
Date: Thu, 7 Nov 2024 12:32:18 +0200
Subject: [PATCH 097/100] composer

---
 composer.lock | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/composer.lock b/composer.lock
index 1a149c62a..517795e54 100644
--- a/composer.lock
+++ b/composer.lock
@@ -840,16 +840,16 @@
         },
         {
             "name": "phpstan/phpstan",
-            "version": "1.12.7",
+            "version": "1.12.8",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpstan/phpstan.git",
-                "reference": "dc2b9976bd8b0f84ec9b0e50cc35378551de7af0"
+                "reference": "f6a60a4d66142b8156c9da923f1972657bc4748c"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpstan/phpstan/zipball/dc2b9976bd8b0f84ec9b0e50cc35378551de7af0",
-                "reference": "dc2b9976bd8b0f84ec9b0e50cc35378551de7af0",
+                "url": "https://api.github.com/repos/phpstan/phpstan/zipball/f6a60a4d66142b8156c9da923f1972657bc4748c",
+                "reference": "f6a60a4d66142b8156c9da923f1972657bc4748c",
                 "shasum": ""
             },
             "require": {
@@ -894,7 +894,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2024-10-18T11:12:07+00:00"
+            "time": "2024-11-06T19:06:49+00:00"
         },
         {
             "name": "phpunit/php-code-coverage",
@@ -2594,5 +2594,5 @@
         "php": ">=8.3"
     },
     "platform-dev": [],
-    "plugin-api-version": "2.6.0"
+    "plugin-api-version": "2.2.0"
 }

From 88b1975c8aed3493f11648621fba3d9a868f069d Mon Sep 17 00:00:00 2001
From: shimon <shimon@appwrite.io>
Date: Thu, 7 Nov 2024 12:40:20 +0200
Subject: [PATCH 098/100] composer

---
 composer.json |  2 +-
 composer.lock | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/composer.json b/composer.json
index 66b0fad6d..a1ca02e4b 100755
--- a/composer.json
+++ b/composer.json
@@ -37,7 +37,7 @@
         "ext-mbstring": "*",
         "php": ">=8.3",
         "utopia-php/framework": "0.33.*",
-        "utopia-php/cache": "0.11.*",
+        "utopia-php/cache": "0.10.*",
         "utopia-php/mongo": "0.3.*"
     },
     "require-dev": {
diff --git a/composer.lock b/composer.lock
index 517795e54..3122dd6b5 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "f7eec4bad737b741ae97c81db0532d29",
+    "content-hash": "0fffc3b6680e12db596e0fb8dd971606",
     "packages": [
         {
             "name": "jean85/pretty-package-versions",
@@ -216,16 +216,16 @@
         },
         {
             "name": "utopia-php/cache",
-            "version": "0.11.0",
+            "version": "0.10.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/cache.git",
-                "reference": "8ebcab5aac7606331cef69b0081f6c9eff2e58bc"
+                "reference": "b22c6eb6d308de246b023efd0fc9758aee8b8247"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/cache/zipball/8ebcab5aac7606331cef69b0081f6c9eff2e58bc",
-                "reference": "8ebcab5aac7606331cef69b0081f6c9eff2e58bc",
+                "url": "https://api.github.com/repos/utopia-php/cache/zipball/b22c6eb6d308de246b023efd0fc9758aee8b8247",
+                "reference": "b22c6eb6d308de246b023efd0fc9758aee8b8247",
                 "shasum": ""
             },
             "require": {
@@ -236,7 +236,7 @@
             },
             "require-dev": {
                 "laravel/pint": "1.2.*",
-                "phpstan/phpstan": "^1.12",
+                "phpstan/phpstan": "1.9.x-dev",
                 "phpunit/phpunit": "^9.3",
                 "vimeo/psalm": "4.13.1"
             },
@@ -260,9 +260,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/cache/issues",
-                "source": "https://github.com/utopia-php/cache/tree/0.11.0"
+                "source": "https://github.com/utopia-php/cache/tree/0.10.2"
             },
-            "time": "2024-11-05T16:53:58+00:00"
+            "time": "2024-06-25T20:36:35+00:00"
         },
         {
             "name": "utopia-php/framework",

From 9d5faac1bc8bd74f3cba6f6181e559dbaa0e9f89 Mon Sep 17 00:00:00 2001
From: shimon <shimon@appwrite.io>
Date: Thu, 7 Nov 2024 12:43:50 +0200
Subject: [PATCH 099/100] composer

---
 composer.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/composer.json b/composer.json
index a1ca02e4b..66b0fad6d 100755
--- a/composer.json
+++ b/composer.json
@@ -37,7 +37,7 @@
         "ext-mbstring": "*",
         "php": ">=8.3",
         "utopia-php/framework": "0.33.*",
-        "utopia-php/cache": "0.10.*",
+        "utopia-php/cache": "0.11.*",
         "utopia-php/mongo": "0.3.*"
     },
     "require-dev": {

From 31571bd6adaa9f6bfb31195c49237456b0742ec7 Mon Sep 17 00:00:00 2001
From: shimon <shimon@appwrite.io>
Date: Thu, 7 Nov 2024 12:44:19 +0200
Subject: [PATCH 100/100] composer

---
 composer.lock | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/composer.lock b/composer.lock
index 3122dd6b5..517795e54 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "0fffc3b6680e12db596e0fb8dd971606",
+    "content-hash": "f7eec4bad737b741ae97c81db0532d29",
     "packages": [
         {
             "name": "jean85/pretty-package-versions",
@@ -216,16 +216,16 @@
         },
         {
             "name": "utopia-php/cache",
-            "version": "0.10.2",
+            "version": "0.11.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/utopia-php/cache.git",
-                "reference": "b22c6eb6d308de246b023efd0fc9758aee8b8247"
+                "reference": "8ebcab5aac7606331cef69b0081f6c9eff2e58bc"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/utopia-php/cache/zipball/b22c6eb6d308de246b023efd0fc9758aee8b8247",
-                "reference": "b22c6eb6d308de246b023efd0fc9758aee8b8247",
+                "url": "https://api.github.com/repos/utopia-php/cache/zipball/8ebcab5aac7606331cef69b0081f6c9eff2e58bc",
+                "reference": "8ebcab5aac7606331cef69b0081f6c9eff2e58bc",
                 "shasum": ""
             },
             "require": {
@@ -236,7 +236,7 @@
             },
             "require-dev": {
                 "laravel/pint": "1.2.*",
-                "phpstan/phpstan": "1.9.x-dev",
+                "phpstan/phpstan": "^1.12",
                 "phpunit/phpunit": "^9.3",
                 "vimeo/psalm": "4.13.1"
             },
@@ -260,9 +260,9 @@
             ],
             "support": {
                 "issues": "https://github.com/utopia-php/cache/issues",
-                "source": "https://github.com/utopia-php/cache/tree/0.10.2"
+                "source": "https://github.com/utopia-php/cache/tree/0.11.0"
             },
-            "time": "2024-06-25T20:36:35+00:00"
+            "time": "2024-11-05T16:53:58+00:00"
         },
         {
             "name": "utopia-php/framework",