Sync books from remote store (based on Fixes for kobo sync and proxy)

composer.json

@@ -135,6 +135,10 @@
       "Composer\\Config::disableProcessTimeout",
       "env XDEBUG_MODE=off ./vendor/bin/phpunit --colors=always"
     ],
+    "test-phpunit-xdebug": [
+      "Composer\\Config::disableProcessTimeout",
+      "env XDEBUG_MODE=debug XDEBUG_TRIGGER=1 ./vendor/bin/phpunit --colors=always"
+    ],
     "test": [
       "@test-phpcs",
       "@test-phpstan",

config/packages/monolog.yaml

@@ -1,8 +1,10 @@
 monolog:
     channels:
         - deprecation # Deprecations are logged in the dedicated "deprecation" channel when it exists
-        - proxy # Proxy logs are logged in the dedicated "proxy" channel when it exists
-        - kobo
+        - kobo_proxy # Proxy logs are logged in the dedicated "proxy" channel when it exists
+        - kobo_http # All request done on AbstractKoboController
+        - kobo_sync # Log specific to Sync endpoint
+        - kobo_kepubify # kepub conversion
 
 
     handlers:
@@ -16,10 +18,10 @@ monolog:
             type: console
             process_psr_3_messages: false
             channels: ["!event", "!doctrine"]
-        proxy:
+        kobo:
             type: rotating_file
             level: debug
-            channels: ["proxy","kobo"]
+            channels: ["kobo_proxy","kobo_http", "kobo_sync", "kobo_kepubify"]
             path: "%kernel.logs_dir%/kobo.%kernel.environment%.log"
             max_files: 10
 

config/packages/security.yaml

@@ -50,7 +50,7 @@ security:
     # Easy way to control access for large sections of your site
     # Note: Only the *first* access control that matches will be used
     access_control:
-        - { path: ^/api/v3/content/checkforchanges, roles: PUBLIC_ACCESS }
+        - { path: ^/api/v3/content/, roles: PUBLIC_ACCESS }
         - { path: ^/login, roles: PUBLIC_ACCESS }
         - { path: ^/logout, roles: ROLE_USER }
         - { path: ^/, roles: ROLE_USER }

config/services.yaml

@@ -16,6 +16,8 @@ parameters:
     KOBO_IMAGE_API_URL:  '%env(KOBO_IMAGE_API_URL)%'
     env(KEPUBIFY_BIN): "/usr/bin/kepubify"
     KEPUBIFY_BIN:  '%env(KEPUBIFY_BIN)%'
+    KOBO_READINGSERVICES_URL: '%env(bool:KOBO_READINGSERVICES_URL)%'
+    env(KOBO_READINGSERVICES_URL): 'https://readingservices.kobo.com'
 services:
     # default configuration for services in *this* file
     _defaults:
@@ -98,6 +100,7 @@ services:
     App\Kobo\Proxy\KoboProxyConfiguration:
       calls:
         - [ setStoreApiUrl, ['%KOBO_API_URL%'] ]
+        - [ setReadingServiceUrl, ['%KOBO_READINGSERVICES_URL%'] ]
         - [ setImageApiUrl, ['%KOBO_IMAGE_API_URL%'] ]
         - [ setEnabled, ['%KOBO_PROXY_ENABLED%'] ]
         - [ setUseProxyEverywhere, ['%KOBO_PROXY_USE_EVERYWHERE%'] ]
@@ -110,6 +113,10 @@ services:
         tags:
         - { name: kernel.event_listener, event: kernel.request, method: onKernelRequest, priority: 101 }
 
+    App\Kobo\LogProcessor\KoboContextProcessor:
+        tags:
+            - { name: monolog.processor }
+
 when@dev:
     services:
         Symfony\Component\HttpKernel\Profiler\Profiler: '@profiler'

migrations/Version20241129185216.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20241129185216 extends AbstractMigration
+{
+    public function getDescription(): string
+    {
+        return 'Add upstream_sync on koboDevice';
+    }
+
+    public function up(Schema $schema): void
+    {
+        $this->addSql('ALTER TABLE kobo_device ADD upstream_sync TINYINT(1) DEFAULT 0 NOT NULL');
+    }
+
+    public function down(Schema $schema): void
+    {
+        $this->addSql('ALTER TABLE kobo_device DROP upstream_sync');
+    }
+}

src/Controller/Kobo/KoboController.php

@@ -41,6 +41,7 @@ public function index(): Response
     #[Route('/v1/products/featured/{uuid}', requirements: ['uuid' => '^[a-zA-Z0-9\-]+$'], methods: ['GET', 'POST'])]
     #[Route('/v1/products/{uuid}/prices', requirements: ['uuid' => '^[a-zA-Z0-9\-]+$'], methods: ['GET', 'POST'])]
     #[Route('/v1/products/{uuid}/recommendations', requirements: ['uuid' => '^[a-zA-Z0-9\-]+$'], methods: ['GET', 'POST'])]
+    #[Route('/v1/user/recommendations/feedback', methods: ['GET', 'POST'])]
     #[Route('/v1/products/{uuid}/reviews', requirements: ['uuid' => '^[a-zA-Z0-9\-]+$'], methods: ['GET', 'POST'])]
     #[Route('/v1/user/profile')]
     #[Route('/v1/configuration')]

src/Controller/Kobo/KoboDeviceController.php

@@ -6,10 +6,8 @@
 use App\Entity\User;
 use App\Form\KoboType;
 use App\Repository\KoboDeviceRepository;
-use Devdot\Monolog\Parser;
 use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
-use Symfony\Component\DependencyInjection\ParameterBag\ParameterBagInterface;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Attribute\Route;
@@ -30,35 +28,6 @@ public function index(KoboDeviceRepository $koboDeviceRepository): Response
         ]);
     }
 
-    #[Route('/logs', name: 'app_kobodevice_user_logs', methods: ['GET'])]
-    public function logs(ParameterBagInterface $parameterBag): Response
-    {
-        if (!$this->getUser() instanceof UserInterface) {
-            throw $this->createAccessDeniedException();
-        }
-
-        $records = [];
-
-        try {
-            $logDir = $parameterBag->get('kernel.logs_dir');
-            $env = $parameterBag->get('kernel.environment');
-
-            if (!is_string($logDir) || !is_string($env)) {
-                throw new \RuntimeException('Invalid log directory or environment');
-            }
-
-            $parser = new Parser($logDir.'/kobo.'.$env.'-'.date('Y-m-d').'.log');
-
-            $records = $parser->get();
-        } catch (\Exception $e) {
-            $this->addFlash('warning', $e->getMessage());
-        }
-
-        return $this->render('kobodevice_user/logs.html.twig', [
-            'records' => $records,
-        ]);
-    }
-
     #[Route('/new', name: 'app_kobodevice_user_new', methods: ['GET', 'POST'])]
     public function new(Request $request, EntityManagerInterface $entityManager): Response
     {

src/Controller/Kobo/KoboDeviceLogsController.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace App\Controller\Kobo;
+
+use Devdot\Monolog\Parser;
+use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\Routing\Attribute\Route;
+use Symfony\Component\Security\Core\User\UserInterface;
+
+#[Route('/user/kobo')]
+class KoboDeviceLogsController extends AbstractController
+{
+    public function __construct(
+        #[Autowire(param: 'kernel.logs_dir')]
+        protected string $kernelLogsDir,
+        #[Autowire(param: 'kernel.environment')]
+        protected string $kernelEnvironment,
+    ) {
+    }
+
+    #[Route('/logs', name: 'app_kobodevice_user_logs', methods: ['GET'])]
+    public function logs(): Response
+    {
+        if (!$this->getUser() instanceof UserInterface) {
+            throw $this->createAccessDeniedException();
+        }
+
+        $records = [];
+
+        try {
+            $parser = new Parser($this->kernelLogsDir.'/kobo.'.$this->kernelEnvironment.'-'.date('Y-m-d').'.log');
+
+            $records = $parser->get();
+        } catch (\Exception $e) {
+            $this->addFlash('warning', $e->getMessage());
+        }
+
+        return $this->render('kobodevice_user/logs.html.twig', [
+            'records' => $records,
+        ]);
+    }
+}

src/Controller/Kobo/KoboInitializationController.php

@@ -11,7 +11,6 @@
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Attribute\Route;
-use Symfony\Component\Routing\Generator\UrlGenerator;
 use Symfony\Component\Routing\Generator\UrlGeneratorInterface;
 
 #[Route('/kobo/{accessKey}')]
@@ -48,20 +47,20 @@ public function initialization(Request $request, KoboDevice $kobo): Response
         $base = rtrim($base, '/');
 
         // Image host: https://<domain>
-        $jsonData['Resources']['image_host'] = rtrim($this->generateUrl('app_dashboard', [], UrlGenerator::ABSOLUTE_URL), '/');
+        $jsonData['Resources']['image_host'] = rtrim($this->generateUrl('app_dashboard', [], UrlGeneratorInterface::ABSOLUTE_URL), '/');
         $jsonData['Resources']['image_url_template'] = $base.'/image/{ImageId}/{width}/{height}/{Quality}/isGreyscale/image.jpg';
         $jsonData['Resources']['image_url_quality_template'] = $base.'/{ImageId}/{width}/{height}/false/image.jpg';
 
-        foreach ($jsonData['Resources'] as $key => $url) {
+        // Original value is "https://readingservices.kobo.com" event if the name is "host", it's an url.
+        $jsonData['Resources']['reading_services_host'] = rtrim($this->generateUrl('app_dashboard', [], UrlGeneratorInterface::ABSOLUTE_URL), '/');
+
+        foreach ($jsonData['Resources'] as &$url) {
             if (!is_string($url)) {
                 continue;
             }
-            $jsonData['Resources'][$key] = str_replace('https://storeapi.kobo.com', $base, $url);
+            $url = str_replace('https://storeapi.kobo.com', $base, $url);
         }
 
-        // Reading services
-        $jsonData['Resources']['reading_services_host'] = str_replace('https://', '', rtrim($this->generateUrl('app_dashboard', [], UrlGenerator::ABSOLUTE_URL), '/'));
-
         $response = new JsonResponse($jsonData);
         $response->headers->set('kobo-api-token', 'e30=');
 

src/Controller/Kobo/KoboStateController.php

@@ -16,7 +16,6 @@
 use App\Service\BookProgressionService;
 use Doctrine\ORM\EntityManagerInterface;
 use GuzzleHttp\Exception\GuzzleException;
-use Psr\Log\LoggerInterface;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
@@ -40,7 +39,7 @@ public function __construct(
      * Update reading state.
      **/
     #[Route('/v1/library/{uuid}/state', name: 'api_endpoint_state_put', requirements: ['uuid' => '^[a-zA-Z0-9\-]+$'], methods: ['PUT'])]
-    public function state(KoboDevice $kobo, string $uuid, Request $request): Response|JsonResponse
+    public function putState(KoboDevice $kobo, string $uuid, Request $request): Response|JsonResponse
     {
         $book = $this->bookRepository->findByUuidAndKoboDevice($uuid, $kobo);
 
@@ -88,7 +87,7 @@ public function state(KoboDevice $kobo, string $uuid, Request $request): Respons
      * @throws GuzzleException
      */
     #[Route('/v1/library/{uuid}/state', name: 'api_endpoint_v1_getstate', requirements: ['uuid' => '^[a-zA-Z0-9\-]+$'], methods: ['GET'])]
-    public function getState(KoboDevice $kobo, string $uuid, Request $request, SyncToken $syncToken, LoggerInterface $logger): Response|JsonResponse
+    public function getState(KoboDevice $kobo, string $uuid, Request $request, SyncToken $syncToken): Response|JsonResponse
     {
         // Get State returns an empty response
         $response = new JsonResponse([]);
@@ -110,8 +109,6 @@ public function getState(KoboDevice $kobo, string $uuid, Request $request, SyncT
 
         $response->setContent($rsResponse);
 
-        $logger->info('Returned Kobo State for book', ['response' => $rsResponse]);
-
         return $response;
     }
 

src/Controller/Kobo/KoboSyncController.php

@@ -8,6 +8,7 @@
 use App\Kobo\Proxy\KoboStoreProxy;
 use App\Kobo\Response\SyncResponseFactory;
 use App\Kobo\SyncToken;
+use App\Kobo\UpstreamSyncMerger;
 use App\Repository\BookRepository;
 use App\Repository\KoboDeviceRepository;
 use App\Repository\KoboSyncedBookRepository;
@@ -31,9 +32,10 @@ public function __construct(
         protected KoboSyncTokenExtractor $koboSyncTokenExtractor,
         protected KoboSyncedBookRepository $koboSyncedBookRepository,
         protected ShelfRepository $shelfRepository,
-        protected LoggerInterface $logger,
+        protected LoggerInterface $koboSyncLogger,
         protected KoboDeviceRepository $koboDeviceRepository,
         protected SyncResponseFactory $syncResponseFactory,
+        protected UpstreamSyncMerger $upstreamSyncMerger,
     ) {
     }
 
@@ -54,13 +56,13 @@ public function apiEndpoint(KoboDevice $kobo, SyncToken $syncToken, Request $req
         $count = $this->koboSyncedBookRepository->countByKoboDevice($kobo);
         if ($forced || $count === 0) {
             if ($forced) {
-                $this->logger->debug('Force sync for Kobo {id}', ['id' => $kobo->getId()]);
+                $this->koboSyncLogger->debug('Force sync for Kobo {id}', ['id' => $kobo->getId()]);
                 $this->koboSyncedBookRepository->deleteAllSyncedBooks($kobo);
                 $kobo->setForceSync(false);
                 $this->koboDeviceRepository->save($kobo);
                 $syncToken->currentDate = new \DateTime('now');
             }
-            $this->logger->debug('First sync for Kobo {id}', ['id' => $kobo->getId()]);
+            $this->koboSyncLogger->debug('First sync for Kobo {id}', ['id' => $kobo->getId()]);
             $syncToken->lastCreated = null;
             $syncToken->lastModified = null;
             $syncToken->tagLastModified = null;
@@ -70,20 +72,23 @@ public function apiEndpoint(KoboDevice $kobo, SyncToken $syncToken, Request $req
         // We fetch a subset of book to sync, based on the SyncToken.
         $books = $this->bookRepository->getChangedBooks($kobo, $syncToken, 0, self::MAX_BOOKS_PER_SYNC);
         $count = $this->bookRepository->getChangedBooksCount($kobo, $syncToken);
-        $this->logger->debug("Sync for Kobo {id}: {$count} books to sync", ['id' => $kobo->getId(), 'count' => $count, 'token' => $syncToken]);
+        $this->koboSyncLogger->debug("Sync for Kobo {id}: {$count} books to sync", ['id' => $kobo->getId(), 'count' => $count, 'token' => $syncToken]);
 
         $response = $this->syncResponseFactory->create($syncToken, $kobo)
             ->addBooks($books)
             ->addShelves($this->shelfRepository->getShelvesToSync($kobo, $syncToken));
 
+        // Fetch the books upstream and merge the answer
+        $shouldContinue = $this->upstreamSyncMerger->merge($kobo, $response, $request);
+
         // TODO Pagination based on the sync token and lastSyncDate
         $httpResponse = $response->toJsonResponse();
-        $httpResponse->headers->set('x-kobo-sync-todo', count($books) < $count ? 'continue' : 'done');
+        $httpResponse->headers->set('x-kobo-sync-todo', $shouldContinue || count($books) < $count ? 'continue' : 'done');
 
         // Once the response is generated, we update the list of synced books
         // If you do this before, the logic will be broken
         if (false === $forced) {
-            $this->logger->debug('Set synced date for {count} downloaded books', ['count' => count($books)]);
+            $this->koboSyncLogger->debug('Set synced date for {count} downloaded books', ['count' => count($books)]);
 
             $this->koboSyncedBookRepository->updateSyncedBooks($kobo, $books, $syncToken);
         }

src/Controller/Kobo/KoboTagController.php

@@ -25,7 +25,7 @@ public function __construct(
         protected ShelfRepository $shelfRepository,
         protected KoboStoreProxy $koboStoreProxy,
         protected SerializerInterface $serializer,
-        protected LoggerInterface $logger)
+        protected LoggerInterface $koboLogger)
     {
     }
 
@@ -44,7 +44,7 @@ public function delete(Request $request, KoboDevice $kobo, string $shelfId): Res
 
             // Avoid the Kobo to send the request over and over again by marking it successful
             if ($response->getStatusCode() === Response::HTTP_NOT_FOUND) {
-                $this->logger->debug('Shelf not found locally and via the proxy, marking deletion as successful anyway', ['shelfId' => $shelfId]);
+                $this->koboLogger->debug('Shelf not found locally and via the proxy, marking deletion as successful anyway', ['shelfId' => $shelfId]);
 
                 return new JsonResponse($shelfId, Response::HTTP_CREATED);
             }
@@ -54,7 +54,7 @@ public function delete(Request $request, KoboDevice $kobo, string $shelfId): Res
 
         /** @var TagDeleteRequest $deleteRequest */
         $deleteRequest = $this->serializer->deserialize($request->getContent(false), TagDeleteRequest::class, 'json');
-        $this->logger->debug('Tag delete request', ['request' => $deleteRequest]);
+        $this->koboLogger->debug('Tag delete request', ['request' => $deleteRequest]);
 
         try {
             if (!$shelf instanceof Shelf) {
@@ -91,14 +91,14 @@ public function tags(Request $request, KoboDevice $kobo, ?string $tagId = null):
 
         if ($request->isMethod('DELETE')) {
             if ($shelf instanceof Shelf) {
-                $this->logger->debug('Removing kobo from shelf', ['shelf' => $shelf, 'kobo' => $kobo]);
+                $this->koboLogger->debug('Removing kobo from shelf', ['shelf' => $shelf, 'kobo' => $kobo]);
                 $shelf->removeKoboDevice($kobo);
                 $this->shelfRepository->flush();
 
                 return new JsonResponse(['deleted'], Response::HTTP_OK);
             }
             if ($this->koboStoreProxy->isEnabled()) {
-                $this->logger->debug('Proxying request to delete tag {id}', ['id' => $tagId]);
+                $this->koboLogger->debug('Proxying request to delete tag {id}', ['id' => $tagId]);
 
                 $proxyResponse = $this->koboStoreProxy->proxy($request);
                 if ($proxyResponse->getStatusCode() === Response::HTTP_NOT_FOUND) {