feat: created clickhouse model in comments entity

docker-compose.yml

@@ -26,7 +26,7 @@ services:
  ports:
  - "6379:6379"
  volumes:
- - ./data/redis:/data
+ - ./db/redis:/db
 
  ch_server:
  image: yandex/clickhouse-server

pyproject.toml

@@ -23,6 +23,7 @@ pytest = "^8.2.0"
 sqlalchemy = "^2.0.31"
 sqlalchemy-clickhouse = "^0.1.5.post0"
 alembic = "^1.13.2"
+clickhouse-sqlalchemy = "^0.3.2"
 
 
 [build-system]

src/recommendation_engine/app/features/comments/db/model.py

@@ -0,0 +1,23 @@
+from typing import TypeVar
+from clickhouse_sqlalchemy import engines
+from sqlalchemy import Column, Integer, String, Date
+
+from ..domain.entity.getir import GetirComments
+from ....core.database.clickhouse import ClickhouseBase
+from ..domain.entity.yemek_sepeti import YemekSepetiComments
+
+
+CommentEntity = TypeVar('CommentEntity', GetirComments, YemekSepetiComments)
+
+
+class CommentsModel(ClickhouseBase):
+ __tablename__ = 'comments'
+
+ @staticmethod
+ def from_entity(entity: CommentEntity):
+ pass
+
+ __table_args__ = (
+ engines.MergeTree(order_by=['id']),
+ {'schema': "default"},
+ )

src/recommendation_engine/app/features/comments/domain/entity/getir.py

@@ -50,14 +50,14 @@ def _iterate_over_comments(self) -> Generator[dict, None, None]:
  if not data:
  break
 
- comments = data.get("data", {}).get("reviews", [])
+ comments = data.get("db", {}).get("reviews", [])
 
  if not comments:
  break
 
  yield comments
  logger.info(
- f"page {skip} was crawled. total crawled data is {len(self.comment_stack)}"
+ f"page {skip} was crawled. total crawled db is {len(self.comment_stack)}"
  )
 
  skip += 10

src/recommendation_engine/app/features/comments/domain/entity/yemek_sepeti.py

@@ -63,15 +63,15 @@ def _iterate_over_comments(self) -> Generator[dict, None, None]:
  if not data:
  break
 
- comments = data.get("data")
+ comments = data.get("db")
  next_page_key = data.get("pageKey", "")
 
  if not comments:
  break
 
  yield comments
  logger.info(
- f"Page was crawled. total crawled data is {len(self.comment_stack)}"
+ f"Page was crawled. total crawled db is {len(self.comment_stack)}"
  )
  if not next_page_key:
  break

src/recommendation_engine/app/features/menu/domain/entity/yemeksepeti.py

@@ -65,7 +65,7 @@ def _retrieve_menu_from_api(self) -> list[dict] | None:
  if not data:
  return []
 
- menu_list = data.get("data", {}).get("menus", [])
+ menu_list = data.get("db", {}).get("menus", [])
  if len(menu_list) > 0:
  menu_list = menu_list[0].get("menu_categories")
  logger.info(f"Menu with {self.restaurant_id} was crawled.")

src/recommendation_engine/app/features/restaurants/domain/entity/getir.py

@@ -66,15 +66,15 @@ def _iterate_over_restaurants(self) -> Generator[dict, None, None]:
  break
 
  restaurants = (
- data.get("data", {}).get("restaurantSection", {}).get("restaurants")
+ data.get("db", {}).get("restaurantSection", {}).get("restaurants")
  )
 
  if not restaurants:
  break
 
  yield restaurants
  logger.info(
- f"page {skip} was crawled. total crawled data is {len(self.restaurant_stack)}"
+ f"page {skip} was crawled. total crawled db is {len(self.restaurant_stack)}"
  )
 
  skip += 10