From ba0f52bc42a0d08e6bd7bcb247e32727ed8c91ff Mon Sep 17 00:00:00 2001 From: fmelihh Date: Sun, 7 Jul 2024 01:22:08 +0300 Subject: [PATCH] feat: project structure was updated. --- .../app/events/kafka/client/__init__.py | 2 - .../app/events/kafka/client/base.py | 18 --- .../app/events/kafka/client/consumer.py | 153 ------------------ .../app/events/kafka/client/producer.py | 61 ------- .../app/events/kafka/config.py | 47 ------ .../app/{domain => features}/__init__.py | 0 .../{domain => features}/comments/__init__.py | 0 .../comments/domain}/__init__.py | 0 .../comments/domain/entity}/__init__.py | 0 .../comments/domain}/entity/getir.py | 8 +- .../comments/domain}/entity/yemek_sepeti.py | 8 +- .../comments/domain}/values/__init__.py | 0 .../comments/domain}/values/comment.py | 2 +- .../comments/domain}/values/getir/__init__.py | 0 .../comments/domain}/values/getir/comment.py | 0 .../domain}/values/yemeksepeti/__init__.py | 0 .../domain}/values/yemeksepeti/comment.py | 0 .../values/yemeksepeti/product_variation.py | 0 .../domain}/values/yemeksepeti/rating.py | 0 .../domain}/values/yemeksepeti/replies.py | 0 .../app/{domain => features}/entity.py | 2 +- .../menu/entity => features/menu}/__init__.py | 0 .../menu/domain}/__init__.py | 0 .../menu/domain/entity}/__init__.py | 0 .../menu/domain}/entity/getir.py | 8 +- .../menu/domain}/entity/yemeksepeti.py | 8 +- .../menu/domain}/values/__init__.py | 0 .../menu/domain}/values/geo.py | 0 .../menu/domain}/values/getir/__init__.py | 0 .../menu/domain}/values/getir/menu.py | 0 .../menu/domain}/values/menu.py | 2 +- .../menu/domain}/values/price.py | 0 .../domain/values/yemeksepeti}/__init__.py | 0 .../menu/domain}/values/yemeksepeti/menu.py | 0 .../app/{domain => features}/mixins.py | 0 .../app/{domain => features}/processor.py | 6 +- .../app/{domain => features}/request.py | 0 .../restaurants}/__init__.py | 0 .../restaurants/domain}/__init__.py | 0 .../restaurants/domain}/entity/__init__.py | 0 .../restaurants/domain}/entity/getir.py | 10 +- .../domain}/entity/yemek_sepeti.py | 10 +- .../restaurants/domain}/values/__init__.py | 0 .../domain}/values/delivery_time.py | 0 .../restaurants/domain}/values/geo.py | 0 .../domain}/values/getir/__init__.py | 0 .../domain}/values/getir/restaurant.py | 0 .../restaurants/domain}/values/price.py | 0 .../domain}/values/rating_count.py | 0 .../restaurants/domain}/values/restaurant.py | 2 +- .../domain}/values/yemeksepeti/__init__.py | 0 .../domain}/values/yemeksepeti/restaurant.py | 0 .../app/{domain => features}/value_stack.py | 6 +- .../app/usecase/commands/__init__.py | 0 .../app/usecase/commands/kafka.py | 64 -------- .../app/usecase/schedulers/__init__.py | 0 .../app/usecase/schedulers/dlq_shovel.py | 15 -- .../domain/comments/test_getir_comments.py | 2 +- .../comments/test_yemeksepeti_comments.py | 2 +- tests/app/domain/menu/test_getir_menu.py | 2 +- .../app/domain/menu/test_yemeksepeti_menu.py | 2 +- .../getir/test_getir_restaurants.py | 4 +- .../test_yemeksepeti_restaurant.py | 4 +- 63 files changed, 44 insertions(+), 404 deletions(-) delete mode 100644 src/recommendation_engine/app/events/kafka/client/__init__.py delete mode 100644 src/recommendation_engine/app/events/kafka/client/base.py delete mode 100644 src/recommendation_engine/app/events/kafka/client/consumer.py delete mode 100644 src/recommendation_engine/app/events/kafka/client/producer.py delete mode 100644 src/recommendation_engine/app/events/kafka/config.py rename src/recommendation_engine/app/{domain => features}/__init__.py (100%) rename src/recommendation_engine/app/{domain => features}/comments/__init__.py (100%) rename src/recommendation_engine/app/{domain/comments/entity => features/comments/domain}/__init__.py (100%) rename src/recommendation_engine/app/{domain/menu => features/comments/domain/entity}/__init__.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/entity/getir.py (94%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/entity/yemek_sepeti.py (96%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/__init__.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/comment.py (70%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/getir/__init__.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/getir/comment.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/yemeksepeti/__init__.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/yemeksepeti/comment.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/yemeksepeti/product_variation.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/yemeksepeti/rating.py (100%) rename src/recommendation_engine/app/{domain/comments => features/comments/domain}/values/yemeksepeti/replies.py (100%) rename src/recommendation_engine/app/{domain => features}/entity.py (90%) rename src/recommendation_engine/app/{domain/menu/entity => features/menu}/__init__.py (100%) rename src/recommendation_engine/app/{domain/menu/values/yemeksepeti => features/menu/domain}/__init__.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/menu/domain/entity}/__init__.py (100%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/entity/getir.py (94%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/entity/yemeksepeti.py (95%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/values/__init__.py (100%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/values/geo.py (100%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/values/getir/__init__.py (100%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/values/getir/menu.py (100%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/values/menu.py (70%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/values/price.py (100%) rename src/recommendation_engine/app/{events => features/menu/domain/values/yemeksepeti}/__init__.py (100%) rename src/recommendation_engine/app/{domain/menu => features/menu/domain}/values/yemeksepeti/menu.py (100%) rename src/recommendation_engine/app/{domain => features}/mixins.py (100%) rename src/recommendation_engine/app/{domain => features}/processor.py (93%) rename src/recommendation_engine/app/{domain => features}/request.py (100%) rename src/recommendation_engine/app/{events/kafka => features/restaurants}/__init__.py (100%) rename src/recommendation_engine/app/{usecase => features/restaurants/domain}/__init__.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/entity/__init__.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/entity/getir.py (95%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/entity/yemek_sepeti.py (98%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/__init__.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/delivery_time.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/geo.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/getir/__init__.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/getir/restaurant.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/price.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/rating_count.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/restaurant.py (71%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/yemeksepeti/__init__.py (100%) rename src/recommendation_engine/app/{domain/restaurants => features/restaurants/domain}/values/yemeksepeti/restaurant.py (100%) rename src/recommendation_engine/app/{domain => features}/value_stack.py (74%) delete mode 100644 src/recommendation_engine/app/usecase/commands/__init__.py delete mode 100644 src/recommendation_engine/app/usecase/commands/kafka.py delete mode 100644 src/recommendation_engine/app/usecase/schedulers/__init__.py delete mode 100644 src/recommendation_engine/app/usecase/schedulers/dlq_shovel.py diff --git a/src/recommendation_engine/app/events/kafka/client/__init__.py b/src/recommendation_engine/app/events/kafka/client/__init__.py deleted file mode 100644 index c708759..0000000 --- a/src/recommendation_engine/app/events/kafka/client/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .producer import KafkaProducer -from .consumer import KafkaConsumer diff --git a/src/recommendation_engine/app/events/kafka/client/base.py b/src/recommendation_engine/app/events/kafka/client/base.py deleted file mode 100644 index 8a49c98..0000000 --- a/src/recommendation_engine/app/events/kafka/client/base.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import TypeVar -from abc import ABC, abstractmethod -from confluent_kafka import Consumer, Producer - -from ..config import AbstractKafkaConfig - -KafkaClient = TypeVar("KafkaClient", Consumer, Producer) - - -class AbstractKafka(ABC): - def __init__(self, configuration: AbstractKafkaConfig): - self._client = None - self.configuration = configuration - - @property - @abstractmethod - def client(self) -> KafkaClient: - pass diff --git a/src/recommendation_engine/app/events/kafka/client/consumer.py b/src/recommendation_engine/app/events/kafka/client/consumer.py deleted file mode 100644 index 19e46b2..0000000 --- a/src/recommendation_engine/app/events/kafka/client/consumer.py +++ /dev/null @@ -1,153 +0,0 @@ -import json -from loguru import logger -from typing import Generator, Any -from confluent_kafka import Consumer -from contextlib import contextmanager - -from ..config import ConsumerConfig -from .producer import KafkaProducer -from .base import AbstractKafka, KafkaClient -from ....usecase.commands.kafka import KafkaPostfix, KafkaHeader - - -class KafkaConsumer(AbstractKafka): - def __init__( - self, - timeout: int = 600, - batch_size: int = 1000, - producer: KafkaProducer | None = None, - # dlq_errors: list[BaseException] | None = None, - ): - self.timeout = timeout - self._producer = producer - self.batch_size = batch_size - # self.dlq_errors = ( - # dlq_errors - # if isinstance(dlq_errors, list) and len(dlq_errors) > 0 - # else [Exception] - # ) - super().__init__(configuration=ConsumerConfig()) - - @property - def producer(self) -> KafkaProducer: - if self._producer is None: - self._producer = KafkaProducer() - return self._producer - - @property - def client(self) -> KafkaClient: - if self._client is None: - self._client = Consumer(self.configuration.retrieve_config_dictionary()) - return self._client - - def rotate_dql_messages( - self, - topic_names: list[str], - headers: list[dict[str, Any]], - data_values: list[dict[str, Any]], - ): - dlq_rotate_hashmap = {} - for topic, header, data_val in zip(topic_names, headers, data_values): - kafka_header = KafkaHeader.retrieve_object_from_kafka_format(header) - - if not kafka_header.dlq_options.is_reach_dlq_retry_to_maximum(): - topic = topic.replace( - f"-{KafkaPostfix.DEAD_LETTER_QUEUE.value}", "" - ).strip() - - if topic not in dlq_rotate_hashmap: - dlq_rotate_hashmap[topic] = {"data": [], "headers": []} - - dlq_rotate_hashmap[topic]["data"].append(data_val) - dlq_rotate_hashmap[topic]["headers"].append(kafka_header) - - for topic, values in dlq_rotate_hashmap.items(): - self.producer.produce( - topic_name=topic, headers=values["headers"], data=values["data"] - ) - - @staticmethod - def _format_consumed_data(data_list: list) -> dict[str, list[dict[str, Any]]]: - headers = [] - topic_names = [] - data_values = [] - for data in data_list: - headers.append( - { - key: value.decode("utf-8") if isinstance(value, bytes) else value - for key, value in dict(data.headers()).items() - } - ) - topic_names.append(data.topic()) - data_values.append(json.loads(data.value().decode("utf-8"))) - - return { - "headers": headers, - "topic_names": topic_names, - "data_values": data_values, - } - - def process_dead_letter_queue_logic( - self, - topic: str, - exception_reason: str, - headers: list[dict[str, Any]], - data_values: list[dict[str, Any]], - ): - formatted_headers = [] - formatted_data_values = [] - - for idx, header in enumerate(headers): - kafka_header = KafkaHeader.retrieve_object_from_kafka_format(data=header) - kafka_header.exception_reason = exception_reason - - if kafka_header.dlq_is_enabled == "false": - continue - - kafka_header.dlq_options.increase_dlq_retry() - - formatted_headers.append(kafka_header) - formatted_data_values.append(data_values[idx]) - - self.producer.produce( - headers=formatted_headers, - data=formatted_data_values, - topic_name=f"{topic}-{KafkaPostfix.DEAD_LETTER_QUEUE.value}", - ) - - @contextmanager - def dlq_context( - self, - topic: str | list[str], - headers: list[dict[str, Any]], - data_values: list[dict[str, Any]], - ) -> Generator: - try: - yield - except Exception as e: - logger.exception( - f"An error occurred while processing consumed datas. error details: {e}" - ) - self.process_dead_letter_queue_logic( - topic=topic, - headers=headers, - data_values=data_values, - exception_reason=str(e), - ) - - def consume( - self, topic: str - ) -> Generator[tuple[list[dict[str, Any]], list[dict[str, Any]]], None, None]: - - self.client.subscribe([topic]) - while True: - data_list = self.client.consume(self.batch_size, timeout=self.timeout) - if not data_list: - break - - formatted_data_list = self._format_consumed_data(data_list) - yield ( - formatted_data_list["topic_names"], - formatted_data_list["headers"], - formatted_data_list["data_values"], - ) diff --git a/src/recommendation_engine/app/events/kafka/client/producer.py b/src/recommendation_engine/app/events/kafka/client/producer.py deleted file mode 100644 index e1e7852..0000000 --- a/src/recommendation_engine/app/events/kafka/client/producer.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -import datetime -from typing import Any -from loguru import logger -from confluent_kafka import Producer -from fastapi.exceptions import HTTPException - -from .base import AbstractKafka -from ..config import ProducerConfig -from ....usecase.commands.kafka import KafkaHeader - - -class KafkaProducer(AbstractKafka): - def __init__(self): - super(KafkaProducer, self).__init__(configuration=ProducerConfig()) - - @property - def client(self) -> Producer: - if self._client is None: - self._client = Producer(self.configuration.retrieve_config_dictionary()) - return self._client - - @staticmethod - def delivery_report(err, msg): - if err is not None: - raise HTTPException( - status_code=500, detail=f"message delivery failed {str(err)}" - ) - - logger.info(f"message delivered to {msg.topic()} [{msg.partition()}]") - - def produce( - self, - topic_name: str, - data: list[dict[str, Any]], - headers: KafkaHeader | list[KafkaHeader], - ): - if isinstance(headers, list) and len(headers) != len(data): - raise ValueError( - "if kafka headers are set as a list, they must be same length as the data." - ) - - list_of_headers = ( - [header.model_dump() for header in headers] - if isinstance(headers, list) - else [headers.model_dump()] * len(data) - ) - timestamp = int(datetime.datetime.now().timestamp()) - - for idx, produce_parameters in enumerate(zip(data, list_of_headers)): - record, headers = produce_parameters - self.client.produce( - topic_name, - value=json.dumps(record), - key=f"{idx}_{timestamp}", - callback=self.delivery_report, - headers=headers, - ) - - self.client.poll(0) - self.client.flush() diff --git a/src/recommendation_engine/app/events/kafka/config.py b/src/recommendation_engine/app/events/kafka/config.py deleted file mode 100644 index 06c2d4d..0000000 --- a/src/recommendation_engine/app/events/kafka/config.py +++ /dev/null @@ -1,47 +0,0 @@ -import dataclasses -from abc import ABC, abstractmethod - - -class AbstractKafkaConfig(ABC): - bootstrap_servers: str = "pkc-12576z.us-west2.gcp.confluent.cloud:9092" - security_protocol: str = "SASL_SSL" - sasl_mechanisms: str = "PLAIN" - sasl_username: str = "KMF2L2K5FY5FSWD6" - sasl_password: str = ( - "LhS2a/phg9FljZYjUCdo9yY+FalusxsqT6T73NHKyXpGyeRs9AKestL70pRy05wN" - ) - - @abstractmethod - def retrieve_config_dictionary(self) -> dict: - pass - - -@dataclasses.dataclass(frozen=True) -class ProducerConfig(AbstractKafkaConfig): - def retrieve_config_dictionary(self) -> dict: - return { - "bootstrap.servers": self.bootstrap_servers, - "security.protocol": self.security_protocol, - "sasl.mechanisms": self.sasl_mechanisms, - "sasl.username": self.sasl_username, - "sasl.password": self.sasl_password, - } - - -@dataclasses.dataclass(frozen=True) -class ConsumerConfig(AbstractKafkaConfig): - session_timeout_ms: str = "45000" - group_id: str = "python-group-1" - auto_offset_reset: str = "earliest" - - def retrieve_config_dictionary(self) -> dict: - return { - "bootstrap.servers": self.bootstrap_servers, - "security.protocol": self.security_protocol, - "sasl.mechanisms": self.sasl_mechanisms, - "sasl.username": self.sasl_username, - "sasl.password": self.sasl_password, - "session.timeout.ms": self.session_timeout_ms, - "group.id": self.group_id, - "auto.offset.reset": self.auto_offset_reset, - } diff --git a/src/recommendation_engine/app/domain/__init__.py b/src/recommendation_engine/app/features/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/__init__.py rename to src/recommendation_engine/app/features/__init__.py diff --git a/src/recommendation_engine/app/domain/comments/__init__.py b/src/recommendation_engine/app/features/comments/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/__init__.py rename to src/recommendation_engine/app/features/comments/__init__.py diff --git a/src/recommendation_engine/app/domain/comments/entity/__init__.py b/src/recommendation_engine/app/features/comments/domain/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/entity/__init__.py rename to src/recommendation_engine/app/features/comments/domain/__init__.py diff --git a/src/recommendation_engine/app/domain/menu/__init__.py b/src/recommendation_engine/app/features/comments/domain/entity/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/__init__.py rename to src/recommendation_engine/app/features/comments/domain/entity/__init__.py diff --git a/src/recommendation_engine/app/domain/comments/entity/getir.py b/src/recommendation_engine/app/features/comments/domain/entity/getir.py similarity index 94% rename from src/recommendation_engine/app/domain/comments/entity/getir.py rename to src/recommendation_engine/app/features/comments/domain/entity/getir.py index 1c90e7c..75ef5e1 100644 --- a/src/recommendation_engine/app/domain/comments/entity/getir.py +++ b/src/recommendation_engine/app/features/comments/domain/entity/getir.py @@ -1,11 +1,11 @@ from loguru import logger from typing import Generator -from ...entity import BaseEntity -from ...request import RequestValue -from ...value_stack import EntityValueStack +from ....entity import BaseEntity +from ....request import RequestValue +from ....value_stack import EntityValueStack from ..values.getir import GetirCommentValue -from ...processor import Processor, SyncCallParams +from ....processor import Processor, SyncCallParams class GetirComments(BaseEntity, Processor): diff --git a/src/recommendation_engine/app/domain/comments/entity/yemek_sepeti.py b/src/recommendation_engine/app/features/comments/domain/entity/yemek_sepeti.py similarity index 96% rename from src/recommendation_engine/app/domain/comments/entity/yemek_sepeti.py rename to src/recommendation_engine/app/features/comments/domain/entity/yemek_sepeti.py index 79ea379..cbbbe24 100644 --- a/src/recommendation_engine/app/domain/comments/entity/yemek_sepeti.py +++ b/src/recommendation_engine/app/features/comments/domain/entity/yemek_sepeti.py @@ -1,10 +1,10 @@ from loguru import logger from typing import Generator -from ...entity import BaseEntity -from ...request import RequestValue -from ...value_stack import EntityValueStack -from ...processor import Processor, SyncCallParams +from ....entity import BaseEntity +from ....request import RequestValue +from ....value_stack import EntityValueStack +from ....processor import Processor, SyncCallParams from ..values.yemeksepeti import YemeksepetiCommentValue diff --git a/src/recommendation_engine/app/domain/comments/values/__init__.py b/src/recommendation_engine/app/features/comments/domain/values/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/__init__.py rename to src/recommendation_engine/app/features/comments/domain/values/__init__.py diff --git a/src/recommendation_engine/app/domain/comments/values/comment.py b/src/recommendation_engine/app/features/comments/domain/values/comment.py similarity index 70% rename from src/recommendation_engine/app/domain/comments/values/comment.py rename to src/recommendation_engine/app/features/comments/domain/values/comment.py index 3765ac0..c5d661d 100644 --- a/src/recommendation_engine/app/domain/comments/values/comment.py +++ b/src/recommendation_engine/app/features/comments/domain/values/comment.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from ...mixins import DataclassValidationMixin +from ....mixins import DataclassValidationMixin @dataclass(frozen=True) diff --git a/src/recommendation_engine/app/domain/comments/values/getir/__init__.py b/src/recommendation_engine/app/features/comments/domain/values/getir/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/getir/__init__.py rename to src/recommendation_engine/app/features/comments/domain/values/getir/__init__.py diff --git a/src/recommendation_engine/app/domain/comments/values/getir/comment.py b/src/recommendation_engine/app/features/comments/domain/values/getir/comment.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/getir/comment.py rename to src/recommendation_engine/app/features/comments/domain/values/getir/comment.py diff --git a/src/recommendation_engine/app/domain/comments/values/yemeksepeti/__init__.py b/src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/yemeksepeti/__init__.py rename to src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/__init__.py diff --git a/src/recommendation_engine/app/domain/comments/values/yemeksepeti/comment.py b/src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/comment.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/yemeksepeti/comment.py rename to src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/comment.py diff --git a/src/recommendation_engine/app/domain/comments/values/yemeksepeti/product_variation.py b/src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/product_variation.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/yemeksepeti/product_variation.py rename to src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/product_variation.py diff --git a/src/recommendation_engine/app/domain/comments/values/yemeksepeti/rating.py b/src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/rating.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/yemeksepeti/rating.py rename to src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/rating.py diff --git a/src/recommendation_engine/app/domain/comments/values/yemeksepeti/replies.py b/src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/replies.py similarity index 100% rename from src/recommendation_engine/app/domain/comments/values/yemeksepeti/replies.py rename to src/recommendation_engine/app/features/comments/domain/values/yemeksepeti/replies.py diff --git a/src/recommendation_engine/app/domain/entity.py b/src/recommendation_engine/app/features/entity.py similarity index 90% rename from src/recommendation_engine/app/domain/entity.py rename to src/recommendation_engine/app/features/entity.py index 57df990..c0af9d4 100644 --- a/src/recommendation_engine/app/domain/entity.py +++ b/src/recommendation_engine/app/features/entity.py @@ -10,5 +10,5 @@ def __eq__(self, value: "BaseEntity") -> bool: if isinstance(value, type(self)): return self.entity_id == value.entity_id - def __hash__(self) -> int: + def __hash__(self) -> str: return self.entity_id diff --git a/src/recommendation_engine/app/domain/menu/entity/__init__.py b/src/recommendation_engine/app/features/menu/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/entity/__init__.py rename to src/recommendation_engine/app/features/menu/__init__.py diff --git a/src/recommendation_engine/app/domain/menu/values/yemeksepeti/__init__.py b/src/recommendation_engine/app/features/menu/domain/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/values/yemeksepeti/__init__.py rename to src/recommendation_engine/app/features/menu/domain/__init__.py diff --git a/src/recommendation_engine/app/domain/restaurants/__init__.py b/src/recommendation_engine/app/features/menu/domain/entity/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/__init__.py rename to src/recommendation_engine/app/features/menu/domain/entity/__init__.py diff --git a/src/recommendation_engine/app/domain/menu/entity/getir.py b/src/recommendation_engine/app/features/menu/domain/entity/getir.py similarity index 94% rename from src/recommendation_engine/app/domain/menu/entity/getir.py rename to src/recommendation_engine/app/features/menu/domain/entity/getir.py index 5306c34..ff403de 100644 --- a/src/recommendation_engine/app/domain/menu/entity/getir.py +++ b/src/recommendation_engine/app/features/menu/domain/entity/getir.py @@ -1,10 +1,10 @@ from loguru import logger -from ...entity import BaseEntity -from ...request import RequestValue +from ....entity import BaseEntity +from ....request import RequestValue from ..values.getir import GetirMenuValue -from ...value_stack import EntityValueStack -from ...processor import Processor, SyncCallParams +from ....value_stack import EntityValueStack +from ....processor import Processor, SyncCallParams class GetirMenu(BaseEntity, Processor): diff --git a/src/recommendation_engine/app/domain/menu/entity/yemeksepeti.py b/src/recommendation_engine/app/features/menu/domain/entity/yemeksepeti.py similarity index 95% rename from src/recommendation_engine/app/domain/menu/entity/yemeksepeti.py rename to src/recommendation_engine/app/features/menu/domain/entity/yemeksepeti.py index 271457a..51c1521 100644 --- a/src/recommendation_engine/app/domain/menu/entity/yemeksepeti.py +++ b/src/recommendation_engine/app/features/menu/domain/entity/yemeksepeti.py @@ -1,10 +1,10 @@ from loguru import logger from ..values import GeoValue -from ...entity import BaseEntity -from ...request import RequestValue -from ...value_stack import EntityValueStack -from ...processor import Processor, SyncCallParams +from ....entity import BaseEntity +from ....request import RequestValue +from ....value_stack import EntityValueStack +from ....processor import Processor, SyncCallParams from ..values.yemeksepeti.menu import YemeksepetiMenuValue diff --git a/src/recommendation_engine/app/domain/menu/values/__init__.py b/src/recommendation_engine/app/features/menu/domain/values/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/values/__init__.py rename to src/recommendation_engine/app/features/menu/domain/values/__init__.py diff --git a/src/recommendation_engine/app/domain/menu/values/geo.py b/src/recommendation_engine/app/features/menu/domain/values/geo.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/values/geo.py rename to src/recommendation_engine/app/features/menu/domain/values/geo.py diff --git a/src/recommendation_engine/app/domain/menu/values/getir/__init__.py b/src/recommendation_engine/app/features/menu/domain/values/getir/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/values/getir/__init__.py rename to src/recommendation_engine/app/features/menu/domain/values/getir/__init__.py diff --git a/src/recommendation_engine/app/domain/menu/values/getir/menu.py b/src/recommendation_engine/app/features/menu/domain/values/getir/menu.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/values/getir/menu.py rename to src/recommendation_engine/app/features/menu/domain/values/getir/menu.py diff --git a/src/recommendation_engine/app/domain/menu/values/menu.py b/src/recommendation_engine/app/features/menu/domain/values/menu.py similarity index 70% rename from src/recommendation_engine/app/domain/menu/values/menu.py rename to src/recommendation_engine/app/features/menu/domain/values/menu.py index c286a4d..de02b3d 100644 --- a/src/recommendation_engine/app/domain/menu/values/menu.py +++ b/src/recommendation_engine/app/features/menu/domain/values/menu.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from ...mixins import DataclassValidationMixin +from ....mixins import DataclassValidationMixin @dataclass(frozen=True) diff --git a/src/recommendation_engine/app/domain/menu/values/price.py b/src/recommendation_engine/app/features/menu/domain/values/price.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/values/price.py rename to src/recommendation_engine/app/features/menu/domain/values/price.py diff --git a/src/recommendation_engine/app/events/__init__.py b/src/recommendation_engine/app/features/menu/domain/values/yemeksepeti/__init__.py similarity index 100% rename from src/recommendation_engine/app/events/__init__.py rename to src/recommendation_engine/app/features/menu/domain/values/yemeksepeti/__init__.py diff --git a/src/recommendation_engine/app/domain/menu/values/yemeksepeti/menu.py b/src/recommendation_engine/app/features/menu/domain/values/yemeksepeti/menu.py similarity index 100% rename from src/recommendation_engine/app/domain/menu/values/yemeksepeti/menu.py rename to src/recommendation_engine/app/features/menu/domain/values/yemeksepeti/menu.py diff --git a/src/recommendation_engine/app/domain/mixins.py b/src/recommendation_engine/app/features/mixins.py similarity index 100% rename from src/recommendation_engine/app/domain/mixins.py rename to src/recommendation_engine/app/features/mixins.py diff --git a/src/recommendation_engine/app/domain/processor.py b/src/recommendation_engine/app/features/processor.py similarity index 93% rename from src/recommendation_engine/app/domain/processor.py rename to src/recommendation_engine/app/features/processor.py index 7b30a79..c19a3f1 100644 --- a/src/recommendation_engine/app/domain/processor.py +++ b/src/recommendation_engine/app/features/processor.py @@ -13,9 +13,9 @@ from typing import TypeVar, List, Literal from pydantic import BaseModel, Field, AnyUrl -from .menu.values import MenuValue -from .comments.values import CommentValue -from .restaurants.values import RestaurantValue +from .menu.domain.values import MenuValue +from .comments.domain.values import CommentValue +from .restaurants.domain.values import RestaurantValue T = TypeVar("T", *[List[RestaurantValue], List[CommentValue], List[MenuValue]]) diff --git a/src/recommendation_engine/app/domain/request.py b/src/recommendation_engine/app/features/request.py similarity index 100% rename from src/recommendation_engine/app/domain/request.py rename to src/recommendation_engine/app/features/request.py diff --git a/src/recommendation_engine/app/events/kafka/__init__.py b/src/recommendation_engine/app/features/restaurants/__init__.py similarity index 100% rename from src/recommendation_engine/app/events/kafka/__init__.py rename to src/recommendation_engine/app/features/restaurants/__init__.py diff --git a/src/recommendation_engine/app/usecase/__init__.py b/src/recommendation_engine/app/features/restaurants/domain/__init__.py similarity index 100% rename from src/recommendation_engine/app/usecase/__init__.py rename to src/recommendation_engine/app/features/restaurants/domain/__init__.py diff --git a/src/recommendation_engine/app/domain/restaurants/entity/__init__.py b/src/recommendation_engine/app/features/restaurants/domain/entity/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/entity/__init__.py rename to src/recommendation_engine/app/features/restaurants/domain/entity/__init__.py diff --git a/src/recommendation_engine/app/domain/restaurants/entity/getir.py b/src/recommendation_engine/app/features/restaurants/domain/entity/getir.py similarity index 95% rename from src/recommendation_engine/app/domain/restaurants/entity/getir.py rename to src/recommendation_engine/app/features/restaurants/domain/entity/getir.py index c7f95cb..3071fab 100644 --- a/src/recommendation_engine/app/domain/restaurants/entity/getir.py +++ b/src/recommendation_engine/app/features/restaurants/domain/entity/getir.py @@ -2,11 +2,11 @@ from typing import Generator from ..values import GeoValue -from ...entity import BaseEntity -from ...processor import Processor -from ...request import RequestValue -from ...processor import SyncCallParams -from ...value_stack import EntityValueStack +from ....entity import BaseEntity +from ....processor import Processor +from ....request import RequestValue +from ....processor import SyncCallParams +from ....value_stack import EntityValueStack from ..values.getir import GetirRestaurantValue diff --git a/src/recommendation_engine/app/domain/restaurants/entity/yemek_sepeti.py b/src/recommendation_engine/app/features/restaurants/domain/entity/yemek_sepeti.py similarity index 98% rename from src/recommendation_engine/app/domain/restaurants/entity/yemek_sepeti.py rename to src/recommendation_engine/app/features/restaurants/domain/entity/yemek_sepeti.py index d15858b..f4bfdd8 100644 --- a/src/recommendation_engine/app/domain/restaurants/entity/yemek_sepeti.py +++ b/src/recommendation_engine/app/features/restaurants/domain/entity/yemek_sepeti.py @@ -2,11 +2,11 @@ from typing import Generator from ..values import GeoValue -from ...entity import BaseEntity -from ...processor import Processor -from ...request import RequestValue -from ...processor import SyncCallParams -from ...value_stack import EntityValueStack +from ....entity import BaseEntity +from ....processor import Processor +from ....request import RequestValue +from ....processor import SyncCallParams +from ....value_stack import EntityValueStack from ..values.yemeksepeti import YemeksepetiRestaurantValue diff --git a/src/recommendation_engine/app/domain/restaurants/values/__init__.py b/src/recommendation_engine/app/features/restaurants/domain/values/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/__init__.py rename to src/recommendation_engine/app/features/restaurants/domain/values/__init__.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/delivery_time.py b/src/recommendation_engine/app/features/restaurants/domain/values/delivery_time.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/delivery_time.py rename to src/recommendation_engine/app/features/restaurants/domain/values/delivery_time.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/geo.py b/src/recommendation_engine/app/features/restaurants/domain/values/geo.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/geo.py rename to src/recommendation_engine/app/features/restaurants/domain/values/geo.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/getir/__init__.py b/src/recommendation_engine/app/features/restaurants/domain/values/getir/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/getir/__init__.py rename to src/recommendation_engine/app/features/restaurants/domain/values/getir/__init__.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/getir/restaurant.py b/src/recommendation_engine/app/features/restaurants/domain/values/getir/restaurant.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/getir/restaurant.py rename to src/recommendation_engine/app/features/restaurants/domain/values/getir/restaurant.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/price.py b/src/recommendation_engine/app/features/restaurants/domain/values/price.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/price.py rename to src/recommendation_engine/app/features/restaurants/domain/values/price.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/rating_count.py b/src/recommendation_engine/app/features/restaurants/domain/values/rating_count.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/rating_count.py rename to src/recommendation_engine/app/features/restaurants/domain/values/rating_count.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/restaurant.py b/src/recommendation_engine/app/features/restaurants/domain/values/restaurant.py similarity index 71% rename from src/recommendation_engine/app/domain/restaurants/values/restaurant.py rename to src/recommendation_engine/app/features/restaurants/domain/values/restaurant.py index 936bf50..4701cbc 100644 --- a/src/recommendation_engine/app/domain/restaurants/values/restaurant.py +++ b/src/recommendation_engine/app/features/restaurants/domain/values/restaurant.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from ...mixins import DataclassValidationMixin +from ....mixins import DataclassValidationMixin @dataclass(frozen=True) diff --git a/src/recommendation_engine/app/domain/restaurants/values/yemeksepeti/__init__.py b/src/recommendation_engine/app/features/restaurants/domain/values/yemeksepeti/__init__.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/yemeksepeti/__init__.py rename to src/recommendation_engine/app/features/restaurants/domain/values/yemeksepeti/__init__.py diff --git a/src/recommendation_engine/app/domain/restaurants/values/yemeksepeti/restaurant.py b/src/recommendation_engine/app/features/restaurants/domain/values/yemeksepeti/restaurant.py similarity index 100% rename from src/recommendation_engine/app/domain/restaurants/values/yemeksepeti/restaurant.py rename to src/recommendation_engine/app/features/restaurants/domain/values/yemeksepeti/restaurant.py diff --git a/src/recommendation_engine/app/domain/value_stack.py b/src/recommendation_engine/app/features/value_stack.py similarity index 74% rename from src/recommendation_engine/app/domain/value_stack.py rename to src/recommendation_engine/app/features/value_stack.py index e57a450..aad87a8 100644 --- a/src/recommendation_engine/app/domain/value_stack.py +++ b/src/recommendation_engine/app/features/value_stack.py @@ -1,8 +1,8 @@ from typing import TypeVar -from .menu.values import MenuValue -from .comments.values import CommentValue -from .restaurants.values import RestaurantValue +from .menu.domain.values import MenuValue +from .comments.domain.values import CommentValue +from .restaurants.domain.values import RestaurantValue Z = TypeVar("Z", RestaurantValue, CommentValue, MenuValue) diff --git a/src/recommendation_engine/app/usecase/commands/__init__.py b/src/recommendation_engine/app/usecase/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/recommendation_engine/app/usecase/commands/kafka.py b/src/recommendation_engine/app/usecase/commands/kafka.py deleted file mode 100644 index 308ff9b..0000000 --- a/src/recommendation_engine/app/usecase/commands/kafka.py +++ /dev/null @@ -1,64 +0,0 @@ -import uuid -from enum import Enum -from typing import Literal, Optional, Any -from pydantic import BaseModel, Field, model_validator - - -class KafkaPostfix(Enum): - DEAD_LETTER_QUEUE = "dlq" - - -class KafkaTopic(Enum): - SEARCH_RECORD = "search-record" - - -class KafkaHeader(BaseModel): - exception_reason: str | None = Field(default=None) - request_uuid: str = Field(default=str(uuid.uuid4())) - dlq_is_enabled: Literal["true", "false"] = Field(default="false") - dlq_options: Optional["KafkaDeadLetterQueueOptions"] = Field(default=None) - - @classmethod - def retrieve_object_from_kafka_format(cls, data: dict[str, Any]) -> "KafkaHeader": - if data["dlq_is_enabled"] == "true": - return cls( - exception_reason=data.get("exception_reason"), - request_uuid=data["request_uuid"], - dlq_is_enabled=data["dlq_is_enabled"], - dlq_options=KafkaDeadLetterQueueOptions( - dlq_retry=data["dlq_retry"], dlq_max_retry=data["dlq_max_retry"] - ), - ) - else: - return cls(**data) - - @model_validator(mode="after") - def validate_kafka_header(self) -> "KafkaHeader": - if self.dlq_is_enabled == "true" and self.dlq_options is None: - raise ValueError( - "When dead letter queue is enabled, you must specify dead letter queue options." - ) - return self - - def model_dump(self, *args, **kwargs) -> dict[str, Any]: - data = super().model_dump(*args, **kwargs) - if self.dlq_is_enabled == "true": - dlq_options = data.pop("dlq_options") - data.update(dlq_options) - - return data - - -class KafkaDeadLetterQueueOptions(BaseModel): - dlq_retry: str = Field(default="0") - dlq_max_retry: str = Field(default="10") - - def increase_dlq_retry(self): - self.dlq_retry = str(int(self.dlq_retry) + 1) - - def is_reach_dlq_retry_to_maximum(self) -> bool: - dlq_retry = int(self.dlq_retry) - dlq_max_retry = int(self.dlq_max_retry) - if dlq_retry >= dlq_max_retry: - return True - return False diff --git a/src/recommendation_engine/app/usecase/schedulers/__init__.py b/src/recommendation_engine/app/usecase/schedulers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/recommendation_engine/app/usecase/schedulers/dlq_shovel.py b/src/recommendation_engine/app/usecase/schedulers/dlq_shovel.py deleted file mode 100644 index 2efffe7..0000000 --- a/src/recommendation_engine/app/usecase/schedulers/dlq_shovel.py +++ /dev/null @@ -1,15 +0,0 @@ -from celery import Task -from ..commands.kafka import KafkaPostfix -from ...events.kafka.client import KafkaConsumer - - -class DlqShovel(Task): - def run(self, *args, **kwargs): - try: - consumer = KafkaConsumer() - topic = f"^.*-{KafkaPostfix.DEAD_LETTER_QUEUE.value}" - - for topic_names, headers, data_values in consumer.consume(topic=topic): - consumer.rotate_dql_messages(topic_names, headers, data_values) - except Exception as e: - raise self.retry(exc=e, countdown=10, raise_exception=True) diff --git a/tests/app/domain/comments/test_getir_comments.py b/tests/app/domain/comments/test_getir_comments.py index 70ebda0..6e5b38c 100644 --- a/tests/app/domain/comments/test_getir_comments.py +++ b/tests/app/domain/comments/test_getir_comments.py @@ -1,4 +1,4 @@ -from src.recommendation_engine.app.domain.comments.entity.getir import ( +from src.recommendation_engine.app.features.comments.domain.entity.getir import ( GetirComments, ) diff --git a/tests/app/domain/comments/test_yemeksepeti_comments.py b/tests/app/domain/comments/test_yemeksepeti_comments.py index ebdd3d0..d6a4039 100644 --- a/tests/app/domain/comments/test_yemeksepeti_comments.py +++ b/tests/app/domain/comments/test_yemeksepeti_comments.py @@ -1,4 +1,4 @@ -from src.recommendation_engine.app.domain.comments.entity.yemek_sepeti import ( +from src.recommendation_engine.app.features.comments.domain.entity.yemek_sepeti import ( YemekSepetiComments, ) diff --git a/tests/app/domain/menu/test_getir_menu.py b/tests/app/domain/menu/test_getir_menu.py index dda61f5..07e69b0 100644 --- a/tests/app/domain/menu/test_getir_menu.py +++ b/tests/app/domain/menu/test_getir_menu.py @@ -1,4 +1,4 @@ -from src.recommendation_engine.app.domain.menu.entity.getir import ( +from src.recommendation_engine.app.features.menu.domain.entity.getir import ( GetirMenu, ) diff --git a/tests/app/domain/menu/test_yemeksepeti_menu.py b/tests/app/domain/menu/test_yemeksepeti_menu.py index cef729d..0e2e134 100644 --- a/tests/app/domain/menu/test_yemeksepeti_menu.py +++ b/tests/app/domain/menu/test_yemeksepeti_menu.py @@ -1,4 +1,4 @@ -from src.recommendation_engine.app.domain.menu.entity.yemeksepeti import ( +from src.recommendation_engine.app.features.menu.domain.entity.yemeksepeti import ( YemeksepetiMenu, GeoValue, ) diff --git a/tests/app/domain/restaurants/getir/test_getir_restaurants.py b/tests/app/domain/restaurants/getir/test_getir_restaurants.py index b4a7040..93689e8 100644 --- a/tests/app/domain/restaurants/getir/test_getir_restaurants.py +++ b/tests/app/domain/restaurants/getir/test_getir_restaurants.py @@ -1,7 +1,7 @@ -from src.recommendation_engine.app.domain.restaurants.values import ( +from src.recommendation_engine.app.features.restaurants.domain.values import ( GeoValue, ) -from src.recommendation_engine.app.domain.restaurants.entity.getir import ( +from src.recommendation_engine.app.features.restaurants.domain.entity.getir import ( GetirRestaurants, ) diff --git a/tests/app/domain/restaurants/yemeksepeti/test_yemeksepeti_restaurant.py b/tests/app/domain/restaurants/yemeksepeti/test_yemeksepeti_restaurant.py index a2345b4..a6538e2 100644 --- a/tests/app/domain/restaurants/yemeksepeti/test_yemeksepeti_restaurant.py +++ b/tests/app/domain/restaurants/yemeksepeti/test_yemeksepeti_restaurant.py @@ -1,7 +1,7 @@ -from src.recommendation_engine.app.domain.restaurants.values import ( +from src.recommendation_engine.app.features.restaurants.domain.values import ( GeoValue, ) -from src.recommendation_engine.app.domain.restaurants.entity.yemek_sepeti import ( +from src.recommendation_engine.app.features.restaurants.domain.entity.yemek_sepeti import ( YemeksepetiRestaurants, )