From 26357da06251538c3bdd12d46efd6fdb7c213faa Mon Sep 17 00:00:00 2001 From: Laurent Constantin Date: Fri, 29 Nov 2024 19:54:17 +0100 Subject: [PATCH] feat(kobo): Sync remote books support --- migrations/Version20241129185216.php | 29 +++++++++ src/Controller/Kobo/KoboSyncController.php | 17 +++-- src/Entity/KoboDevice.php | 13 ++++ src/Form/KoboType.php | 12 +++- src/Kobo/Response/SyncResponse.php | 20 ++++++ src/Kobo/UpstreamSyncMerger.php | 76 ++++++++++++++++++++++ 6 files changed, 159 insertions(+), 8 deletions(-) create mode 100644 migrations/Version20241129185216.php create mode 100644 src/Kobo/UpstreamSyncMerger.php diff --git a/migrations/Version20241129185216.php b/migrations/Version20241129185216.php new file mode 100644 index 00000000..155dc80b --- /dev/null +++ b/migrations/Version20241129185216.php @@ -0,0 +1,29 @@ +addSql('ALTER TABLE kobo_device ADD upstream_sync TINYINT(1) DEFAULT 0 NOT NULL'); + } + + public function down(Schema $schema): void + { + $this->addSql('ALTER TABLE kobo_device DROP upstream_sync'); + } +} diff --git a/src/Controller/Kobo/KoboSyncController.php b/src/Controller/Kobo/KoboSyncController.php index c3cc49b0..1791e48e 100644 --- a/src/Controller/Kobo/KoboSyncController.php +++ b/src/Controller/Kobo/KoboSyncController.php @@ -8,6 +8,7 @@ use App\Kobo\Proxy\KoboStoreProxy; use App\Kobo\Response\SyncResponseFactory; use App\Kobo\SyncToken; +use App\Kobo\UpstreamSyncMerger; use App\Repository\BookRepository; use App\Repository\KoboDeviceRepository; use App\Repository\KoboSyncedBookRepository; @@ -31,9 +32,10 @@ public function __construct( protected KoboSyncTokenExtractor $koboSyncTokenExtractor, protected KoboSyncedBookRepository $koboSyncedBookRepository, protected ShelfRepository $shelfRepository, - protected LoggerInterface $logger, + protected LoggerInterface $koboSyncLogger, protected KoboDeviceRepository $koboDeviceRepository, protected SyncResponseFactory $syncResponseFactory, + protected UpstreamSyncMerger $upstreamSyncMerger, ) { } @@ -54,13 +56,13 @@ public function apiEndpoint(KoboDevice $kobo, SyncToken $syncToken, Request $req $count = $this->koboSyncedBookRepository->countByKoboDevice($kobo); if ($forced || $count === 0) { if ($forced) { - $this->logger->debug('Force sync for Kobo {id}', ['id' => $kobo->getId()]); + $this->koboSyncLogger->debug('Force sync for Kobo {id}', ['id' => $kobo->getId()]); $this->koboSyncedBookRepository->deleteAllSyncedBooks($kobo); $kobo->setForceSync(false); $this->koboDeviceRepository->save($kobo); $syncToken->currentDate = new \DateTime('now'); } - $this->logger->debug('First sync for Kobo {id}', ['id' => $kobo->getId()]); + $this->koboSyncLogger->debug('First sync for Kobo {id}', ['id' => $kobo->getId()]); $syncToken->lastCreated = null; $syncToken->lastModified = null; $syncToken->tagLastModified = null; @@ -70,20 +72,23 @@ public function apiEndpoint(KoboDevice $kobo, SyncToken $syncToken, Request $req // We fetch a subset of book to sync, based on the SyncToken. $books = $this->bookRepository->getChangedBooks($kobo, $syncToken, 0, self::MAX_BOOKS_PER_SYNC); $count = $this->bookRepository->getChangedBooksCount($kobo, $syncToken); - $this->logger->debug("Sync for Kobo {id}: {$count} books to sync", ['id' => $kobo->getId(), 'count' => $count, 'token' => $syncToken]); + $this->koboSyncLogger->debug("Sync for Kobo {id}: {$count} books to sync", ['id' => $kobo->getId(), 'count' => $count, 'token' => $syncToken]); $response = $this->syncResponseFactory->create($syncToken, $kobo) ->addBooks($books) ->addShelves($this->shelfRepository->getShelvesToSync($kobo, $syncToken)); + // Fetch the books upstream and merge the answer + $shouldContinue = $this->upstreamSyncMerger->merge($kobo, $response, $request); + // TODO Pagination based on the sync token and lastSyncDate $httpResponse = $response->toJsonResponse(); - $httpResponse->headers->set('x-kobo-sync-todo', count($books) < $count ? 'continue' : 'done'); + $httpResponse->headers->set('x-kobo-sync-todo', $shouldContinue || count($books) < $count ? 'continue' : 'done'); // Once the response is generated, we update the list of synced books // If you do this before, the logic will be broken if (false === $forced) { - $this->logger->debug('Set synced date for {count} downloaded books', ['count' => count($books)]); + $this->koboSyncLogger->debug('Set synced date for {count} downloaded books', ['count' => count($books)]); $this->koboSyncedBookRepository->updateSyncedBooks($kobo, $books, $syncToken); } diff --git a/src/Entity/KoboDevice.php b/src/Entity/KoboDevice.php index 934d3e5b..005ef814 100644 --- a/src/Entity/KoboDevice.php +++ b/src/Entity/KoboDevice.php @@ -59,6 +59,9 @@ class KoboDevice #[ORM\Column(type: 'boolean', options: ['default' => false])] private bool $forceSync = false; + #[ORM\Column(type: 'boolean', options: ['default' => false])] + private bool $upstreamSync = false; + public function __construct() { $this->shelves = new ArrayCollection(); @@ -195,4 +198,14 @@ public function setModel(?string $model): self return $this; } + + public function isUpstreamSync(): bool + { + return $this->upstreamSync; + } + + public function setUpstreamSync(bool $upstreamSync): void + { + $this->upstreamSync = $upstreamSync; + } } diff --git a/src/Form/KoboType.php b/src/Form/KoboType.php index fc238419..f0c10b70 100644 --- a/src/Form/KoboType.php +++ b/src/Form/KoboType.php @@ -4,6 +4,7 @@ use App\Entity\KoboDevice; use App\Entity\Shelf; +use App\Kobo\Proxy\KoboProxyConfiguration; use Doctrine\ORM\EntityRepository; use Doctrine\ORM\QueryBuilder; use Symfony\Bridge\Doctrine\Form\Type\EntityType; @@ -14,8 +15,10 @@ class KoboType extends AbstractType { - public function __construct(protected Security $security) - { + public function __construct( + protected Security $security, + protected KoboProxyConfiguration $koboProxyConfiguration, + ) { } public function buildForm(FormBuilderInterface $builder, array $options): void @@ -36,6 +39,11 @@ public function buildForm(FormBuilderInterface $builder, array $options): void ->add('forceSync', null, [ 'label' => 'Force Sync', 'required' => false, + ]) + ->add('upstreamSync', null, [ + 'label' => 'Sync books with the official store too', + 'required' => false, + 'disabled' => !$this->koboProxyConfiguration->useProxy(), ]); $builder->add('shelves', EntityType::class, [ 'label' => 'Sync with Shelves', diff --git a/src/Kobo/Response/SyncResponse.php b/src/Kobo/Response/SyncResponse.php index 0a0492f3..e341a608 100644 --- a/src/Kobo/Response/SyncResponse.php +++ b/src/Kobo/Response/SyncResponse.php @@ -18,6 +18,8 @@ * @phpstan-type BookMetadata array * @phpstan-type BookReadingState array> * @phpstan-type BookTag array + * @phpstan-type RemoteItem array + * @phpstan-type RemoteItems array */ class SyncResponse { @@ -33,6 +35,11 @@ class SyncResponse public const READING_STATUS_IN_PROGRESS = 'Reading'; private SyncResponseHelper $helper; + /** + * @var RemoteItems + */ + private array $remoteItems = []; + public function __construct( protected MetadataResponseService $metadataResponse, protected BookProgressionService $bookProgressionService, @@ -52,6 +59,9 @@ public function toJsonResponse(): JsonResponse array_push($list, ...$this->getChangedReadingState()); array_push($list, ...$this->getNewTags()); array_push($list, ...$this->getChangedTag()); + + $list = array_merge($list, $this->remoteItems); + array_filter($list); $response = new JsonResponse(); @@ -245,4 +255,14 @@ private function getChangedReadingState(): array return $response; }, $books); } + + /** + * @param RemoteItems $items + */ + public function addRemoteItems(array $items): self + { + $this->remoteItems = array_merge($this->remoteItems, $items); + + return $this; + } } diff --git a/src/Kobo/UpstreamSyncMerger.php b/src/Kobo/UpstreamSyncMerger.php new file mode 100644 index 00000000..492dea51 --- /dev/null +++ b/src/Kobo/UpstreamSyncMerger.php @@ -0,0 +1,76 @@ +isUpstreamSync() || false === $this->koboStoreProxy->isEnabled()) { + $this->koboSyncLogger->debug('Your device {device} has "upstream sync" disabled', [ + 'device' => $device->getId(), + ]); + + return false; + } + + $response = $this->koboStoreProxy->proxy($request, ['stream' => false]); + if (false === $response->isOk()) { + $this->koboSyncLogger->error('Sync response is not ok. Got '.$response->getStatusCode()); + + return false; + } + + $json = $this->parseJson($response); + + if ($json === []) { + return false; + } + + $this->koboSyncLogger->info('Merging {count} upstream items', [ + 'device' => $device->getId(), + 'count' => count($json), + ]); + $syncResponse->addRemoteItems($json); + + return $this->shouldContinue($response); + } + + private function parseJson(Response $response): array + { + try { + $result = $response->getContent(); + if (false === $result) { + throw new \RuntimeException('Response content is false. Code: '.$response->getStatusCode()); + } + + return (array) json_decode($result, true, 512, JSON_THROW_ON_ERROR); + } catch (\Throwable $e) { + $this->koboSyncLogger->warning('Unable to upstream sync response: {content}', [ + 'exception' => $e, + 'content' => $response->getContent(), + ]); + + return []; + } + } + + private function shouldContinue(Response $response): bool + { + return $response->headers->get('x-kobo-sync') === 'continue'; + } +}