From 5b80d62097d4b7ce201b3b916e65893269864f8b Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 14 May 2024 16:58:37 +0900 Subject: [PATCH 1/7] Fix file memory unfreed and file status not being set --- MigrationCLI.php | 78 +++++++++++++++++++++++++ composer.json | 2 +- playground.php | 2 +- src/Migration/Destinations/Appwrite.php | 4 ++ src/Migration/Transfer.php | 2 +- 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 MigrationCLI.php diff --git a/MigrationCLI.php b/MigrationCLI.php new file mode 100644 index 0000000..419b3f3 --- /dev/null +++ b/MigrationCLI.php @@ -0,0 +1,78 @@ +transfer->getStatusCounters(); + + $mask = "| %15.15s | %-7.7s | %10.10s | %7.7s | %7.7s | %8.8s |\n"; + printf($mask, 'Resource', 'Pending', 'Processing', 'Skipped', 'Warning', 'Success'); + printf($mask, '-------------', '-------------', '-------------', '-------------', '-------------', '-------------'); + foreach ($statusCounters as $resource => $data) { + printf($mask, $resource, $data['pending'], $data['processing'], $data['skip'], $data['warning'], $data['success']); + } + } + + function start() + { + $dotenv = Dotenv::createImmutable(__DIR__); + $dotenv->load(); + + /** + * Initialise All Source Adapters + */ + $source = new Appwrite( + $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], + $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], + $_ENV['SOURCE_APPWRITE_TEST_KEY'] + ); + + // $source->report(); + + $destination = new DestinationsAppwrite( + $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], + $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], + $_ENV['DESTINATION_APPWRITE_TEST_KEY'] + ); + + /** + * Initialise Transfer Class + */ + $this->transfer = new Transfer( + $source, + $destination + ); + + /** + * Run Transfer + */ + $this->transfer->run( + $source->getSupportedResources(), + function (array $resources) { + $this->drawFrame(); + } + ); + + var_dump('Complete'); + } +} + +$instance = new MigrationCLI(); +$instance->start(); +$instance->drawFrame(); diff --git a/composer.json b/composer.json index d943d28..b5b9999 100644 --- a/composer.json +++ b/composer.json @@ -19,7 +19,7 @@ }, "require": { "php": "8.*", - "utopia-php/cli": "0.*", + "utopia-php/cli": "^0.18.0", "appwrite/appwrite": "10.1.0" }, "require-dev": { diff --git a/playground.php b/playground.php index 571a768..4685c13 100644 --- a/playground.php +++ b/playground.php @@ -99,4 +99,4 @@ function (array $resources) use ($transfer) { /* @var \Utopia\Migration\Exception $error */ Console::error('[Destination] ['.$error->getResourceType().'] '.$error->getMessage()); } -} +} \ No newline at end of file diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 2cf8465..a1316d3 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -504,9 +504,13 @@ public function importFile(File $file): File if ($response['signature'] !== $file->getSignature()) { $file->setStatus(Resource::STATUS_WARNING, 'File signature mismatch, Possibly corrupted.'); } + } else { + $file->setStatus(Resource::STATUS_SUCCESS); + $file->setData(''); } } + $file->setData(''); return $file; } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 6049ec9..a1a5ebc 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -95,7 +95,7 @@ public function getStatusCounters() foreach ($this->cache->getAll() as $resources) { foreach ($resources as $resource) { - /** @var resource $resource */ + /** @var Resource $resource */ if (isset($status[$resource->getName()])) { $status[$resource->getName()][$resource->getStatus()]++; if ($status[$resource->getName()]['pending'] > 0) { From 3f2adb385ee5d551fa6db74c03c4cd0205a72528 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 14 May 2024 16:59:26 +0900 Subject: [PATCH 2/7] Remove CLI Package --- composer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/composer.json b/composer.json index b5b9999..2ea999e 100644 --- a/composer.json +++ b/composer.json @@ -19,7 +19,6 @@ }, "require": { "php": "8.*", - "utopia-php/cli": "^0.18.0", "appwrite/appwrite": "10.1.0" }, "require-dev": { From 2adfc089d1253bcfe8989835ea1ccd20486be91d Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Tue, 14 May 2024 17:04:32 +0900 Subject: [PATCH 3/7] Run Linter --- MigrationCLI.php | 9 ++++----- playground.php | 2 +- src/Migration/Destinations/Appwrite.php | 1 + src/Migration/Transfer.php | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/MigrationCLI.php b/MigrationCLI.php index 419b3f3..6d1a6e7 100644 --- a/MigrationCLI.php +++ b/MigrationCLI.php @@ -1,23 +1,22 @@ transfer->getStatusCounters(); @@ -29,7 +28,7 @@ function drawFrame() } } - function start() + public function start() { $dotenv = Dotenv::createImmutable(__DIR__); $dotenv->load(); diff --git a/playground.php b/playground.php index 4685c13..571a768 100644 --- a/playground.php +++ b/playground.php @@ -99,4 +99,4 @@ function (array $resources) use ($transfer) { /* @var \Utopia\Migration\Exception $error */ Console::error('[Destination] ['.$error->getResourceType().'] '.$error->getMessage()); } -} \ No newline at end of file +} diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index a1316d3..4e4970d 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -511,6 +511,7 @@ public function importFile(File $file): File } $file->setData(''); + return $file; } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index a1a5ebc..6049ec9 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -95,7 +95,7 @@ public function getStatusCounters() foreach ($this->cache->getAll() as $resources) { foreach ($resources as $resource) { - /** @var Resource $resource */ + /** @var resource $resource */ if (isset($status[$resource->getName()])) { $status[$resource->getName()][$resource->getStatus()]++; if ($status[$resource->getName()]['pending'] > 0) { From 1b662551edaecf16cffed12a186a5d4678bda041 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 15 May 2024 13:18:47 +0900 Subject: [PATCH 4/7] Address Jake's comments --- MigrationCLI.php | 3 +++ src/Migration/Destinations/Appwrite.php | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/MigrationCLI.php b/MigrationCLI.php index 6d1a6e7..26b14ce 100644 --- a/MigrationCLI.php +++ b/MigrationCLI.php @@ -14,6 +14,9 @@ class MigrationCLI { protected Transfer $transfer; + /** + * Prints the current status of migrations as a table after wiping the screen + */ public function drawFrame() { echo chr(27).chr(91).'H'.chr(27).chr(91).'J'; diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 4e4970d..513602e 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -506,12 +506,10 @@ public function importFile(File $file): File } } else { $file->setStatus(Resource::STATUS_SUCCESS); - $file->setData(''); } } $file->setData(''); - return $file; } From a3cbafd51f00f3f966fce2eb6237677aec59b84f Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 15 May 2024 13:45:47 +0900 Subject: [PATCH 5/7] Move MigrationCLI to bin folder --- MigrationCLI.php | 80 ------------------------------------------------ 1 file changed, 80 deletions(-) delete mode 100644 MigrationCLI.php diff --git a/MigrationCLI.php b/MigrationCLI.php deleted file mode 100644 index 26b14ce..0000000 --- a/MigrationCLI.php +++ /dev/null @@ -1,80 +0,0 @@ -transfer->getStatusCounters(); - - $mask = "| %15.15s | %-7.7s | %10.10s | %7.7s | %7.7s | %8.8s |\n"; - printf($mask, 'Resource', 'Pending', 'Processing', 'Skipped', 'Warning', 'Success'); - printf($mask, '-------------', '-------------', '-------------', '-------------', '-------------', '-------------'); - foreach ($statusCounters as $resource => $data) { - printf($mask, $resource, $data['pending'], $data['processing'], $data['skip'], $data['warning'], $data['success']); - } - } - - public function start() - { - $dotenv = Dotenv::createImmutable(__DIR__); - $dotenv->load(); - - /** - * Initialise All Source Adapters - */ - $source = new Appwrite( - $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], - $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], - $_ENV['SOURCE_APPWRITE_TEST_KEY'] - ); - - // $source->report(); - - $destination = new DestinationsAppwrite( - $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], - $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], - $_ENV['DESTINATION_APPWRITE_TEST_KEY'] - ); - - /** - * Initialise Transfer Class - */ - $this->transfer = new Transfer( - $source, - $destination - ); - - /** - * Run Transfer - */ - $this->transfer->run( - $source->getSupportedResources(), - function (array $resources) { - $this->drawFrame(); - } - ); - - var_dump('Complete'); - } -} - -$instance = new MigrationCLI(); -$instance->start(); -$instance->drawFrame(); From 95a8fb7e17b4ac58c9aea0d21a7b20a900644f62 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 15 May 2024 13:46:37 +0900 Subject: [PATCH 6/7] Move Migration CLI --- bin/MigrationCLI.php | 80 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 bin/MigrationCLI.php diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php new file mode 100644 index 0000000..665eea4 --- /dev/null +++ b/bin/MigrationCLI.php @@ -0,0 +1,80 @@ +transfer->getStatusCounters(); + + $mask = "| %15.15s | %-7.7s | %10.10s | %7.7s | %7.7s | %8.8s |\n"; + printf($mask, 'Resource', 'Pending', 'Processing', 'Skipped', 'Warning', 'Success'); + printf($mask, '-------------', '-------------', '-------------', '-------------', '-------------', '-------------'); + foreach ($statusCounters as $resource => $data) { + printf($mask, $resource, $data['pending'], $data['processing'], $data['skip'], $data['warning'], $data['success']); + } + } + + public function start() + { + $dotenv = Dotenv::createImmutable(__DIR__); + $dotenv->load(); + + /** + * Initialise All Source Adapters + */ + $source = new Appwrite( + $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], + $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], + $_ENV['SOURCE_APPWRITE_TEST_KEY'] + ); + + // $source->report(); + + $destination = new DestinationsAppwrite( + $_ENV['DESTINATION_APPWRITE_TEST_PROJECT'], + $_ENV['DESTINATION_APPWRITE_TEST_ENDPOINT'], + $_ENV['DESTINATION_APPWRITE_TEST_KEY'] + ); + + /** + * Initialise Transfer Class + */ + $this->transfer = new Transfer( + $source, + $destination + ); + + /** + * Run Transfer + */ + $this->transfer->run( + $source->getSupportedResources(), + function (array $resources) { + $this->drawFrame(); + } + ); + + var_dump('Complete'); + } +} + +$instance = new MigrationCLI(); +$instance->start(); +$instance->drawFrame(); \ No newline at end of file From 9ce03145c515d2aefc83926405d888a3ec142369 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Wed, 15 May 2024 13:47:17 +0900 Subject: [PATCH 7/7] Run Linter --- bin/MigrationCLI.php | 4 ++-- src/Migration/Destinations/Appwrite.php | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 665eea4..891b92b 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -1,6 +1,6 @@ start(); -$instance->drawFrame(); \ No newline at end of file +$instance->drawFrame(); diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 513602e..b4bb316 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -510,6 +510,7 @@ public function importFile(File $file): File } $file->setData(''); + return $file; }