feat(kobo): Sync remote books support

migrations/Version20241129185216.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20241129185216 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return 'Add upstream_sync on koboDevice';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql('ALTER TABLE kobo_device ADD upstream_sync TINYINT(1) DEFAULT 0 NOT NULL');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('ALTER TABLE kobo_device DROP upstream_sync');
+    }
+}

src/Controller/Kobo/KoboSyncController.php

@@ -8,6 +8,7 @@
 use App\Kobo\Proxy\KoboStoreProxy;
 use App\Kobo\Response\SyncResponseFactory;
 use App\Kobo\SyncToken;
+use App\Kobo\UpstreamSyncMerger;
 use App\Repository\BookRepository;
 use App\Repository\KoboDeviceRepository;
 use App\Repository\KoboSyncedBookRepository;
@@ -31,9 +32,10 @@ public function __construct(
         protected KoboSyncTokenExtractor $koboSyncTokenExtractor,
         protected KoboSyncedBookRepository $koboSyncedBookRepository,
         protected ShelfRepository $shelfRepository,
-        protected LoggerInterface $logger,
+        protected LoggerInterface $koboSyncLogger,
         protected KoboDeviceRepository $koboDeviceRepository,
         protected SyncResponseFactory $syncResponseFactory,
+        protected UpstreamSyncMerger $upstreamSyncMerger,
     ) {
     }
 
@@ -54,13 +56,13 @@ public function apiEndpoint(KoboDevice $kobo, SyncToken $syncToken, Request $req
         $count = $this->koboSyncedBookRepository->countByKoboDevice($kobo);
         if ($forced || $count === 0) {
             if ($forced) {
-                $this->logger->debug('Force sync for Kobo {id}', ['id' => $kobo->getId()]);
+                $this->koboSyncLogger->debug('Force sync for Kobo {id}', ['id' => $kobo->getId()]);
                 $this->koboSyncedBookRepository->deleteAllSyncedBooks($kobo);
                 $kobo->setForceSync(false);
                 $this->koboDeviceRepository->save($kobo);
                 $syncToken->currentDate = new \DateTime('now');
             }
-            $this->logger->debug('First sync for Kobo {id}', ['id' => $kobo->getId()]);
+            $this->koboSyncLogger->debug('First sync for Kobo {id}', ['id' => $kobo->getId()]);
             $syncToken->lastCreated = null;
             $syncToken->lastModified = null;
             $syncToken->tagLastModified = null;
@@ -70,20 +72,23 @@ public function apiEndpoint(KoboDevice $kobo, SyncToken $syncToken, Request $req
         // We fetch a subset of book to sync, based on the SyncToken.
         $books = $this->bookRepository->getChangedBooks($kobo, $syncToken, 0, self::MAX_BOOKS_PER_SYNC);
         $count = $this->bookRepository->getChangedBooksCount($kobo, $syncToken);
-        $this->logger->debug("Sync for Kobo {id}: {$count} books to sync", ['id' => $kobo->getId(), 'count' => $count, 'token' => $syncToken]);
+        $this->koboSyncLogger->debug("Sync for Kobo {id}: {$count} books to sync", ['id' => $kobo->getId(), 'count' => $count, 'token' => $syncToken]);
 
         $response = $this->syncResponseFactory->create($syncToken, $kobo)
             ->addBooks($books)
             ->addShelves($this->shelfRepository->getShelvesToSync($kobo, $syncToken));
 
+        // Fetch the books upstream and merge the answer
+        $shouldContinue = $this->upstreamSyncMerger->merge($kobo, $response, $request);
+
         // TODO Pagination based on the sync token and lastSyncDate
         $httpResponse = $response->toJsonResponse();
-        $httpResponse->headers->set('x-kobo-sync-todo', count($books) < $count ? 'continue' : 'done');
+        $httpResponse->headers->set('x-kobo-sync-todo', $shouldContinue || count($books) < $count ? 'continue' : 'done');
 
         // Once the response is generated, we update the list of synced books
         // If you do this before, the logic will be broken
         if (false === $forced) {
-            $this->logger->debug('Set synced date for {count} downloaded books', ['count' => count($books)]);
+            $this->koboSyncLogger->debug('Set synced date for {count} downloaded books', ['count' => count($books)]);
 
             $this->koboSyncedBookRepository->updateSyncedBooks($kobo, $books, $syncToken);
         }

src/Entity/KoboDevice.php

@@ -59,6 +59,9 @@ class KoboDevice
     #[ORM\Column(type: 'boolean', options: ['default' => false])]
     private bool $forceSync = false;
 
+    #[ORM\Column(type: 'boolean', options: ['default' => false])]
+    private bool $upstreamSync = false;
+
     public function __construct()
     {
         $this->shelves = new ArrayCollection();
@@ -195,4 +198,14 @@ public function setModel(?string $model): self
 
         return $this;
     }
+
+    public function isUpstreamSync(): bool
+    {
+        return $this->upstreamSync;
+    }
+
+    public function setUpstreamSync(bool $upstreamSync): void
+    {
+        $this->upstreamSync = $upstreamSync;
+    }
 }

src/Form/KoboType.php

@@ -4,6 +4,7 @@
 
 use App\Entity\KoboDevice;
 use App\Entity\Shelf;
+use App\Kobo\Proxy\KoboProxyConfiguration;
 use Doctrine\ORM\EntityRepository;
 use Doctrine\ORM\QueryBuilder;
 use Symfony\Bridge\Doctrine\Form\Type\EntityType;
@@ -14,8 +15,10 @@
 
 class KoboType extends AbstractType
 {
-    public function __construct(protected Security $security)
-    {
+    public function __construct(
+        protected Security $security,
+        protected KoboProxyConfiguration $koboProxyConfiguration,
+    ) {
     }
 
     public function buildForm(FormBuilderInterface $builder, array $options): void
@@ -36,6 +39,11 @@ public function buildForm(FormBuilderInterface $builder, array $options): void
             ->add('forceSync', null, [
                 'label' => 'Force Sync',
                 'required' => false,
+            ])
+            ->add('upstreamSync', null, [
+                'label' => 'Sync books with the official store too',
+                'required' => false,
+                'disabled' => !$this->koboProxyConfiguration->useProxy(),
             ]);
         $builder->add('shelves', EntityType::class, [
             'label' => 'Sync with Shelves',

src/Kobo/Response/SyncResponse.php

@@ -18,6 +18,8 @@
  * @phpstan-type BookMetadata array<string, mixed>
  * @phpstan-type BookReadingState array<int,array<string, mixed>>
  * @phpstan-type BookTag array<string, mixed>
+ * @phpstan-type RemoteItem array<int, object>
+ * @phpstan-type RemoteItems array<int, RemoteItem>
  */
 class SyncResponse
 {
@@ -33,6 +35,11 @@ class SyncResponse
     public const READING_STATUS_IN_PROGRESS = 'Reading';
     private SyncResponseHelper $helper;
 
+    /**
+     * @var RemoteItems
+     */
+    private array $remoteItems = [];
+
     public function __construct(
         protected MetadataResponseService $metadataResponse,
         protected BookProgressionService $bookProgressionService,
@@ -52,6 +59,9 @@ public function toJsonResponse(): JsonResponse
         array_push($list, ...$this->getChangedReadingState());
         array_push($list, ...$this->getNewTags());
         array_push($list, ...$this->getChangedTag());
+
+        $list = $this->remoteItems;
+
         array_filter($list);
 
         $response = new JsonResponse();
@@ -245,4 +255,14 @@ private function getChangedReadingState(): array
             return $response;
         }, $books);
     }
+
+    /**
+     * @param RemoteItems $items
+     */
+    public function addRemoteItems(array $items): self
+    {
+        $this->remoteItems = array_merge($this->remoteItems, $items);
+
+        return $this;
+    }
 }

src/Kobo/UpstreamSyncMerger.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace App\Kobo;
+
+use App\Entity\KoboDevice;
+use App\Kobo\Proxy\KoboStoreProxy;
+use App\Kobo\Response\SyncResponse;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+
+class UpstreamSyncMerger
+{
+    public function __construct(
+        private readonly KoboStoreProxy $koboStoreProxy,
+        private readonly LoggerInterface $koboSyncLogger,
+    ) {
+    }
+
+    public function merge(KoboDevice $device, SyncResponse $syncResponse, Request $request): bool
+    {
+        // Make sure the merge is enabled
+        if (false === $device->isUpstreamSync() || false === $this->koboStoreProxy->isEnabled()) {
+            $this->koboSyncLogger->debug('Your device {device} has "upstream sync" disabled', [
+                'device' => $device->getId(),
+            ]);
+
+            return false;
+        }
+
+        $response = $this->koboStoreProxy->proxy($request, ['stream' => false]);
+        if (false === $response->isOk()) {
+            $this->koboSyncLogger->error('Sync response is not ok. Got '.$response->getStatusCode());
+
+            return false;
+        }
+
+        $json = $this->parseJson($response);
+
+        if ($json === []) {
+            return false;
+        }
+
+        $this->koboSyncLogger->info('Merging {count} upstream items', [
+            'device' => $device->getId(),
+            'count' => count($json),
+        ]);
+        $syncResponse->addRemoteItems($json);
+
+        return $this->shouldContinue($response);
+    }
+
+    private function parseJson(Response $response): array
+    {
+        try {
+            $result = $response->getContent();
+            if (false === $result) {
+                throw new \RuntimeException('Response content is false. Code: '.$response->getStatusCode());
+            }
+
+            return (array) json_decode($result, true, 512, JSON_THROW_ON_ERROR);
+        } catch (\Throwable $e) {
+            $this->koboSyncLogger->warning('Unable to upstream sync response: {content}', [
+                'exception' => $e,
+                'content' => $response->getContent(),
+            ]);
+
+            return [];
+        }
+    }
+
+    private function shouldContinue(Response $response): bool
+    {
+        return $response->headers->get('x-kobo-sync') === 'continue';
+    }
+}