-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit fe8b0a1
Showing
73 changed files
with
2,320 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,169 @@ | ||
name: cinemax-notifications | ||
|
||
on: | ||
push: | ||
branches: [ main ] | ||
|
||
jobs: | ||
tests: | ||
name: Tests | ||
runs-on: ubuntu-latest | ||
|
||
strategy: | ||
matrix: | ||
python-version: ['3.8', '3.9', '3.10'] | ||
|
||
services: | ||
postgres: | ||
image: postgres:14.5-alpine | ||
ports: | ||
- 5432:5432 | ||
env: | ||
POSTGRES_DB: notifications_database | ||
POSTGRES_USER: postgres | ||
POSTGRES_PASSWORD: postgres | ||
POSTGRES_HOST: postgres | ||
POSTGRES_PORT: 5432 | ||
options: >- | ||
--health-cmd "pg_isready" | ||
--health-interval 10s | ||
--health-timeout 5s | ||
--health-retries 5 | ||
redis: | ||
image: redis:7.0.5 | ||
ports: | ||
- 6379:6379 | ||
options: >- | ||
--health-cmd "redis-cli ping || exit 1" | ||
--health-interval 10s | ||
--health-timeout 5s | ||
--health-retries 5 | ||
kafka: | ||
image: confluentinc/cp-kafka:7.3.1 | ||
ports: | ||
- 29092:29092 | ||
env: | ||
KAFKA_BROKER_ID: 1 | ||
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 | ||
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 | ||
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT | ||
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT | ||
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 | ||
KAFKA_AUTO_CREATE_TOPICS_ENABLE: true | ||
options: >- | ||
--health-cmd "nc -z localhost 9092 || exit -1" | ||
--health-interval 5s | ||
--health-timeout 10s | ||
--health-retries 10 | ||
zookeeper: | ||
image: zookeeper:3.8 | ||
options: >- | ||
--health-cmd "nc -z localhost 2181 || exit -1" | ||
--health-interval 5s | ||
--health-timeout 10s | ||
--health-retries 10 | ||
steps: | ||
- name: Check out the repo | ||
uses: actions/checkout@v3 | ||
- name: Set up Python | ||
uses: actions/setup-python@v4 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
- name: Install dependencies | ||
run: | | ||
pip install --upgrade pip | ||
pip install -r backend/django_admin/requirements.txt --no-cache-dir | ||
pip install -r backend/fastapi_receiver/requirements.txt --no-cache-dir | ||
pip install -r backend/faust_worker/requirements.txt --no-cache-dir | ||
- name: Lint with flake8 | ||
run: | | ||
pip install wemake-python-styleguide flake8-html | ||
flake8 backend --format=html --htmldir=flake8 | ||
- name: Lint with mypy | ||
run: | | ||
pip install mypy types-requests lxml | ||
mypy backend --html-report=mypy | ||
- name: Run Admin Panel | ||
run: | | ||
cd backend/django_admin/src | ||
nohup python manage.py runserver & | ||
- name: Stop Admin Panel | ||
run: | | ||
kill $(ps aux | grep python | grep manage.py | awk '{print $2}') | ||
- name: Run API | ||
run: | | ||
cd backend/fastapi_receiver/src | ||
nohup python main.py & | ||
- name: Stop API | ||
run: | | ||
kill $(ps aux | grep python | grep main.py | awk '{print $2}') | ||
- name: Run Worker | ||
run: | | ||
cd backend/faust_worker/src | ||
nohup python main.py worker & | ||
- name: Stop Worker | ||
run: | | ||
kill $(ps aux | grep python | grep main.py | awk '{print $2}') | ||
- name: Output results | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: Report | ||
path: | | ||
flake8/ | ||
mypy/ | ||
docker: | ||
name: Docker | ||
runs-on: ubuntu-latest | ||
needs: tests | ||
steps: | ||
- name: Check out the repo | ||
uses: actions/checkout@v2 | ||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@v1 | ||
- name: Login to Docker | ||
uses: docker/login-action@v1 | ||
with: | ||
username: ${{ secrets.DOCKER_USERNAME }} | ||
password: ${{ secrets.DOCKER_PASSWORD }} | ||
- name: Push Backend to Docker Hub | ||
uses: docker/build-push-action@v2 | ||
with: | ||
push: true | ||
context: backend/django_admin | ||
tags: | | ||
${{ secrets.DOCKER_USERNAME }}/notifications_admin:${{ vars.TAG }} | ||
${{ secrets.DOCKER_USERNAME }}/notifications_admin:latest | ||
- name: Push Backend to Docker Hub | ||
uses: docker/build-push-action@v2 | ||
with: | ||
push: true | ||
context: backend/fastapi_receiver | ||
tags: | | ||
${{ secrets.DOCKER_USERNAME }}/notifications_api:${{ vars.TAG }} | ||
${{ secrets.DOCKER_USERNAME }}/notifications_api:latest | ||
- name: Push Backend to Docker Hub | ||
uses: docker/build-push-action@v2 | ||
with: | ||
push: true | ||
context: backend/faust_worker | ||
tags: | | ||
${{ secrets.DOCKER_USERNAME }}/notifications_worker:${{ vars.TAG }} | ||
${{ secrets.DOCKER_USERNAME }}/notifications_worker:latest | ||
send_message: | ||
name: Send message | ||
runs-on: ubuntu-latest | ||
needs: docker | ||
steps: | ||
- name: Send message | ||
uses: appleboy/telegram-action@master | ||
with: | ||
to: ${{ secrets.TELEGRAM_TO }} | ||
token: ${{ secrets.TELEGRAM_TOKEN }} | ||
message: | | ||
В репозитории ${{ github.repository }} выполнен коммит: | ||
Автор: ${{ github.event.commits[0].author.name }} | ||
Сообщение: ${{ github.event.commits[0].message }} | ||
Ссылка: https://github.com/${{ github.repository }}/commit/${{github.sha}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
.vscode | ||
.DS_Store | ||
.python-version | ||
__pycache__ | ||
.env | ||
main.log | ||
.pytest_cache | ||
.mypy_cache | ||
nohup.out | ||
Worker-data | ||
mypy | ||
flake8 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
## Cinemax Notifications | ||
|
||
[![python](https://img.shields.io/static/v1?label=python&message=3.8%20|%203.9%20|%203.10&color=informational)](https://github.com/temirovazat/cinemax-notifications/actions/workflows/main.yml) | ||
[![dockerfile](https://img.shields.io/static/v1?label=dockerfile&message=published&color=2CB3E8)](https://hub.docker.com/search?q=temirovazat%2Fnotifications) | ||
[![lint](https://img.shields.io/static/v1?label=lint&message=flake8%20|%20mypy&color=brightgreen)](https://github.com/temirovazat/cinemax-notifications/actions/workflows/main.yml) | ||
[![code style](https://img.shields.io/static/v1?label=code%20style&message=WPS&color=orange)](https://wemake-python-styleguide.readthedocs.io/en/latest/) | ||
[![platform](https://img.shields.io/static/v1?label=platform&message=linux%20|%20macos&color=inactive)](https://github.com/temirovazat/cinemax-notifications/actions/workflows/main.yml) | ||
|
||
### **Description** | ||
|
||
_The aim of this project is to implement a notification service for an online cinema. This has led to the development of a system composed of multiple microservices. The notification source is an API designed for receiving events using the [FastAPI](https://fastapi.tiangolo.com) framework. The process responsible for sending notifications (worker) is implemented using the stream processing library [Faust](https://faust.readthedocs.io). Communication between the API and the worker takes place through the message queue [Kafka](https://kafka.apache.org). To create manual notification dispatch, an admin panel based on the [Django](https://www.djangoproject.com) framework is used in conjunction with [Celery](https://docs.celeryq.dev) for sending periodic notifications (scheduler). The admin panel and scheduler interact with the PostgreSQL database in which notifications, their sending history, and execution frequency are stored._ | ||
|
||
### **Technologies** | ||
|
||
```Python``` ```FastAPI``` ```Django``` ```Celery``` ```Faust``` ```Kafka``` ```PostgreSQL``` ```Redis``` ```NGINX``` ```Docker``` | ||
|
||
### **How to Run the Project:** | ||
|
||
Clone the repository and navigate to the `/infra` directory: | ||
``` | ||
git clone https://github.com/temirovazat/cinemax-notifications.git | ||
``` | ||
``` | ||
cd cinemax-notifications/infra/ | ||
``` | ||
|
||
Create a `.env` file and add project settings: | ||
``` | ||
nano .env | ||
``` | ||
``` | ||
# PostgreSQL | ||
POSTGRES_DB=notifications_database | ||
POSTGRES_USER=postgres | ||
POSTGRES_PASSWORD=postgres | ||
POSTGRES_HOST=postgres | ||
POSTGRES_PORT=5432 | ||
# Kafka | ||
KAFKA_HOST=kafka | ||
KAFKA_PORT=9092 | ||
# Redis | ||
REDIS_HOST=redis | ||
REDIS_PORT=6379 | ||
# Django | ||
DJANGO_SUPERUSER_USERNAME=admin | ||
DJANGO_SUPERUSER_EMAIL=admin@mail.ru | ||
DJANGO_SUPERUSER_PASSWORD=1234 | ||
DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,[::1],django | ||
DJANGO_SECRET_KEY=django-insecure-_o)z83b+i@jfjzbof_jn9#%dw*5q2yy3r6zzq-3azof#(vkf!# | ||
# Microservices | ||
EVENT_SOURCING_URL=fastapi:8000 | ||
ADMIN_URL=django:8000 | ||
``` | ||
|
||
Deploy and run the project in containers: | ||
``` | ||
docker-compose up | ||
``` | ||
|
||
Access the admin panel and use the login (admin) and password (1234): | ||
``` | ||
http://127.0.0.1/notifications | ||
``` | ||
|
||
The API documentation will be available at: | ||
``` | ||
http://127.0.0.1/openapi | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
FROM python:3.10 | ||
|
||
WORKDIR /opt/notifications_admin | ||
|
||
COPY requirements.txt requirements.txt | ||
|
||
RUN pip install --upgrade pip \ | ||
&& pip install -r requirements.txt --no-cache-dir | ||
|
||
COPY ./src . | ||
|
||
EXPOSE 8000 | ||
|
||
COPY script.sh / | ||
|
||
RUN chmod +x /script.sh | ||
|
||
ENTRYPOINT ["/script.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
Django==4.1.7 | ||
django-celery-beat==2.4.0 | ||
redis==4.5.1 | ||
python-dotenv==1.0.0 | ||
django-split-settings==1.2.0 | ||
gunicorn==20.1.0 | ||
psycopg2==2.9.3 | ||
requests==2.28.2 | ||
djangorestframework==3.14.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
#!/bin/bash | ||
postgres_ready() { | ||
$(which curl) http://${POSTGRES_HOST:-localhost}:${POSTGRES_PORT:-5432}/ 2>&1 | grep '52' | ||
} | ||
|
||
redis_ready() { | ||
exec 3<>/dev/tcp/${REDIS_HOST:-localhost}/${REDIS_PORT:-6379} && echo -e "PING\r\n" >&3 && head -c 7 <&3 | grep 'PONG' | ||
} | ||
|
||
until redis_ready; do | ||
>&2 echo 'Waiting for Redis to become available...' | ||
sleep 1 | ||
done | ||
>&2 echo 'Redis is available.' | ||
|
||
until postgres_ready; do | ||
>&2 echo 'Waiting for PostgreSQL to become available...' | ||
sleep 1 | ||
done | ||
>&2 echo 'PostgreSQL is available.' | ||
|
||
python manage.py migrate | ||
python manage.py collectstatic --noinput | ||
python manage.py createsuperuser --noinput | ||
gunicorn core.wsgi:application --bind 0:8000 |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
from django.urls import include, path | ||
|
||
urlpatterns = [ | ||
path('v1/', include('api.v1.urls')), | ||
] |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
from typing import Dict | ||
|
||
from rest_framework import serializers | ||
|
||
from notifications.models import Notification, User, UserNotification | ||
|
||
|
||
class UserSerializer(serializers.ModelSerializer): | ||
"""Serializer for the user model.""" | ||
|
||
user_notification = serializers.PrimaryKeyRelatedField(read_only=True) | ||
|
||
class Meta: | ||
"""Metadata.""" | ||
|
||
fields = ('id', 'email', 'delivery_method', 'user_notification') | ||
model = User | ||
|
||
|
||
class UserNotificationSerializer(serializers.ModelSerializer): | ||
"""Serializer for the user notification model.""" | ||
|
||
class Meta: | ||
"""Metadata.""" | ||
|
||
fields = ('id', 'notification', 'user', 'was_sent') | ||
model = UserNotification | ||
|
||
|
||
class UserNotificationCreateSerializer(serializers.ModelSerializer): | ||
"""Serializer for creating user notifications.""" | ||
|
||
email = serializers.CharField(write_only=True) | ||
title = serializers.CharField(write_only=True) | ||
text = serializers.CharField(write_only=True) | ||
|
||
class Meta: | ||
"""Metadata.""" | ||
|
||
fields = ('id', 'email', 'title', 'text') | ||
model = UserNotification | ||
|
||
def create(self, validated_data: Dict) -> UserNotification: | ||
"""Override data creation to create notification and user objects. | ||
Args: | ||
validated_data: Data after validation. | ||
Returns: | ||
UserNotification: User notification. | ||
""" | ||
email = validated_data.pop('email') | ||
data = { | ||
'notification': Notification.objects.create(**validated_data), | ||
'user': User.objects.create(email=email), | ||
} | ||
return super().create(data) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
from django.urls import path | ||
|
||
from api.v1 import views | ||
|
||
urlpatterns = [ | ||
path('users_notifications/', views.UserNotificationCreate.as_view()), | ||
path('users_notifications/<uuid:pk>/', views.UserNotificationUpdate.as_view()), | ||
path('notifications/<uuid:pk>/users/', views.UsersListByNotification.as_view()), | ||
] |
Oops, something went wrong.