From 3576d7384b1b3bba304b734a1d91c816e972cfc9 Mon Sep 17 00:00:00 2001 From: "Konstantin Shperling (Toksi)" <79082640+Toksi86@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:48:16 +0500 Subject: [PATCH 1/4] Relocated openpyxl config file to configs folder (#549) --- .../core/{settings => config/openpyxl}/__init__.py | 0 .../openpyxl_settings.py => config/openpyxl/settings.py} | 0 adaptive_hockey_federation/core/utils.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename adaptive_hockey_federation/core/{settings => config/openpyxl}/__init__.py (100%) rename adaptive_hockey_federation/core/{settings/openpyxl_settings.py => config/openpyxl/settings.py} (100%) diff --git a/adaptive_hockey_federation/core/settings/__init__.py b/adaptive_hockey_federation/core/config/openpyxl/__init__.py similarity index 100% rename from adaptive_hockey_federation/core/settings/__init__.py rename to adaptive_hockey_federation/core/config/openpyxl/__init__.py diff --git a/adaptive_hockey_federation/core/settings/openpyxl_settings.py b/adaptive_hockey_federation/core/config/openpyxl/settings.py similarity index 100% rename from adaptive_hockey_federation/core/settings/openpyxl_settings.py rename to adaptive_hockey_federation/core/config/openpyxl/settings.py diff --git a/adaptive_hockey_federation/core/utils.py b/adaptive_hockey_federation/core/utils.py index c082279a..29879683 100644 --- a/adaptive_hockey_federation/core/utils.py +++ b/adaptive_hockey_federation/core/utils.py @@ -3,7 +3,7 @@ from typing import Any, List, Optional from core.constants import AgeLimits, FileConstants, TimeFormat -from core.settings.openpyxl_settings import ( +from core.config.openpyxl.settings import ( ALIGNMENT_CENTER, HEADERS_BORDER, HEADERS_FILL, From ab46d69fe1471071a617b2dfd0058f7290aeff7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD?= <128288828+InKLaR1TY@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:11:02 +0300 Subject: [PATCH 2/4] Add commands export-db and import-db. (#548) --- Makefile | 12 +++++ .../core/config/dev_settings.py | 5 +- .../core/management/commands/export-db.py | 32 +++++++++++ .../core/management/commands/import-db.py | 54 +++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 adaptive_hockey_federation/core/management/commands/export-db.py create mode 100644 adaptive_hockey_federation/core/management/commands/import-db.py diff --git a/Makefile b/Makefile index 1dc8c8ea..83527318 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,8 @@ help: @echo " run - $(SHELL_GREEN)Команда для локального запуска проекта.$(SHELL_NC)." @echo " fill-db - $(SHELL_GREEN)Команда для заполнения базы данных реальными данными из json фикстур.$(SHELL_NC)." @echo " fill-test-db - $(SHELL_GREEN)Команда для заполнения базы данных тестовыми данными при помощи фабрик генерации данных.$(SHELL_NC)." + @echo " export-db - $(SHELL_GREEN)Команда для экспорта данных из БД в JSON файл.$(SHELL_NC)." + @echo " import-db - $(SHELL_GREEN)Команда для импорта данных из JSON файла в БД.$(SHELL_NC)." @echo " pytest - $(SHELL_GREEN)Команда для прогона юнит тестов pytest.$(SHELL_NC)." @echo " shell - $(SHELL_GREEN)Команда для запуска Django-shell_plus.$(SHELL_NC)." @echo " ds-mock - $(SHELL_GREEN)Команда для запуска имитации DS сервера (порт 8010).$(SHELL_NC)." @@ -117,6 +119,16 @@ fill-test-db: cd $(PROJECT_DIR) && $(DJANGO_RUN) fill-test-db --json-player-data --amount 10 +# Экспорт данных из бд +export-db: + cd $(DJANGO_DIR) && $(DJANGO_RUN) export-db + + +# Импорт данных в бд +import-db: + cd $(DJANGO_DIR) && $(DJANGO_RUN) import-db + + # Прогон тестов с помощью pytest. pytest: cd $(DJANGO_DIR) && pytest diff --git a/adaptive_hockey_federation/core/config/dev_settings.py b/adaptive_hockey_federation/core/config/dev_settings.py index 89eae34e..868093c5 100644 --- a/adaptive_hockey_federation/core/config/dev_settings.py +++ b/adaptive_hockey_federation/core/config/dev_settings.py @@ -29,12 +29,15 @@ } DJANGO_SUPERUSER_USERNAME = env("DJANGO_SUPERUSER_USERNAME", default="admin") -DJANGO_SUPERUSER_EMAIL = env("DJANGO_SUPERUSER_EMAIL", default="admin@admin.ru") +DJANGO_SUPERUSER_EMAIL = env( + "DJANGO_SUPERUSER_EMAIL", default="admin@admin.ru" +) DJANGO_SUPERUSER_PASSWORD = env("DJANGO_SUPERUSER_PASSWORD", default="admin") FIXSTURES_DIR = BASE_DIR / "core" / "fixtures" JSON_PARSER_FILE = "data.json" FIXSTURES_FILE = FIXSTURES_DIR / JSON_PARSER_FILE +DB_DUMP_FILE = FIXSTURES_DIR / "db_dump.json" RESOURSES_ROOT = BASE_DIR / "resourses" # Важен порядок ключей для вставки/удаления diff --git a/adaptive_hockey_federation/core/management/commands/export-db.py b/adaptive_hockey_federation/core/management/commands/export-db.py new file mode 100644 index 00000000..f4b88dba --- /dev/null +++ b/adaptive_hockey_federation/core/management/commands/export-db.py @@ -0,0 +1,32 @@ +from django.core.management import call_command +from django.core.management.base import BaseCommand + +from adaptive_hockey_federation.core.config.dev_settings import DB_DUMP_FILE + + +class Command(BaseCommand): + """Класс экспорта данных из БД.""" + + help = "Экспорт данных из базы данных в JSON-файл" + + def handle(self, *args, **options): + """Экспортирует данные из БД.""" + try: + call_command( + "dumpdata", + exclude=["contenttypes", "auth.permission"], + natural_foreign=True, + natural_primary=True, + indent=2, + output=DB_DUMP_FILE, + ) + + return self.stdout.write( + self.style.SUCCESS( + f"База данных экспортирована в {DB_DUMP_FILE}", + ), + ) + except Exception as e: + return self.stdout.write( + self.style.ERROR(f"Ошибка при экспорте данных: {str(e)}"), + ) diff --git a/adaptive_hockey_federation/core/management/commands/import-db.py b/adaptive_hockey_federation/core/management/commands/import-db.py new file mode 100644 index 00000000..bff63edf --- /dev/null +++ b/adaptive_hockey_federation/core/management/commands/import-db.py @@ -0,0 +1,54 @@ +import json +import os + +from django.apps import apps +from django.core.management import call_command +from django.core.management.base import BaseCommand +from django.db import connection + +from adaptive_hockey_federation.core.config.dev_settings import DB_DUMP_FILE + + +class Command(BaseCommand): + """Класс импорта данных в БД.""" + + help = "Импорт данных из JSON-файла в базу данных" + + def handle(self, *args, **options): + """Импортирует данные в БД.""" + if not os.path.exists(DB_DUMP_FILE): + return self.stdout.write( + self.style.ERROR(f"Файл {DB_DUMP_FILE} не найден"), + ) + + with open(DB_DUMP_FILE, "r") as f: + data = json.load(f) + models_to_clear = set(item["model"] for item in data) + + # Очистка таблиц, в которые импортируются данные, + # для исключения ошибки unique constraint + with connection.cursor() as cursor: + for model_name in models_to_clear: + try: + app_label, model = model_name.split(".") + model_class = apps.get_model(app_label, model) + table_name = model_class._meta.db_table + cursor.execute(f"TRUNCATE TABLE {table_name} CASCADE") + except Exception as e: + return self.stdout.write( + self.style.WARNING( + f"Не удалось очистить таблицу для модели {model_name}: {str(e)}", # noqa: E501 + ), + ) + + try: + call_command("loaddata", DB_DUMP_FILE) + return self.stdout.write( + self.style.SUCCESS( + f"Данные из {DB_DUMP_FILE} импортированы в базу данных", + ), + ) + except Exception as e: + return self.stdout.write( + self.style.ERROR(f"Ошибка при импорте данных: {str(e)}"), + ) From 31d71adf0ac3151eca33345b3e15b757d9a138b9 Mon Sep 17 00:00:00 2001 From: Toksi86 Date: Fri, 27 Sep 2024 16:38:08 +0500 Subject: [PATCH 3/4] Removed the video_api module and everything related to it --- .../core/config/base_settings.py | 1 - .../core/ydisk_utils/__init__.py | 0 .../core/ydisk_utils/utils.py | 104 ----------- adaptive_hockey_federation/games/signals.py | 4 - adaptive_hockey_federation/games/urls.py | 5 - adaptive_hockey_federation/games/views.py | 75 +------- .../main/controllers/player_views.py | 158 +---------------- adaptive_hockey_federation/main/urls.py | 5 - .../service/a_hockey_requests.py | 27 --- .../templates/main/games/game_detail.html | 161 +++++++++--------- .../video_api/__init__.py | 0 adaptive_hockey_federation/video_api/apps.py | 8 - .../video_api/permissions.py | 11 -- .../video_api/serializers.py | 119 ------------- adaptive_hockey_federation/video_api/tasks.py | 150 ---------------- 15 files changed, 82 insertions(+), 746 deletions(-) delete mode 100644 adaptive_hockey_federation/core/ydisk_utils/__init__.py delete mode 100644 adaptive_hockey_federation/core/ydisk_utils/utils.py delete mode 100644 adaptive_hockey_federation/video_api/__init__.py delete mode 100644 adaptive_hockey_federation/video_api/apps.py delete mode 100644 adaptive_hockey_federation/video_api/permissions.py delete mode 100644 adaptive_hockey_federation/video_api/serializers.py delete mode 100644 adaptive_hockey_federation/video_api/tasks.py diff --git a/adaptive_hockey_federation/core/config/base_settings.py b/adaptive_hockey_federation/core/config/base_settings.py index ba753a99..c7302041 100644 --- a/adaptive_hockey_federation/core/config/base_settings.py +++ b/adaptive_hockey_federation/core/config/base_settings.py @@ -41,7 +41,6 @@ "analytics.apps.AnalyticsConfig", "unloads.apps.UnloadsConfig", "games.apps.GamesConfig", - "video_api.apps.VideoApiConfig", ] INSTALLED_APPS = EXTERNAL_APPS + DEFAULT_APPS + LOCAL_APPS diff --git a/adaptive_hockey_federation/core/ydisk_utils/__init__.py b/adaptive_hockey_federation/core/ydisk_utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/adaptive_hockey_federation/core/ydisk_utils/utils.py b/adaptive_hockey_federation/core/ydisk_utils/utils.py deleted file mode 100644 index 091e078d..00000000 --- a/adaptive_hockey_federation/core/ydisk_utils/utils.py +++ /dev/null @@ -1,104 +0,0 @@ -import os -import logging -import sys -from functools import wraps - -import yadisk -from yadisk import Client -from django.conf import settings -from yadisk.exceptions import ( - ForbiddenError, - PathNotFoundError, - ResourceIsLockedError, -) - -from core.constants import YadiskDirectory - - -logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) - -console_handler = logging.StreamHandler(sys.stdout) -console_handler.setLevel(logging.WARNING) - -formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s", -) -console_handler.setFormatter(formatter) - -logger.addHandler(console_handler) - - -def yadisk_client(func): - """ - Декоратор для работы с клиентом Yandex Disk. - - Декорируемая функция должна принимать первым аргументом экземпляр - `yadisk.Client`. - - Пример использования: - @yadisk_client - def function(client, *args, **kwargs): - pass - """ - - @wraps(func) - def wrapper(*args, **kwargs): - try: - with yadisk.Client( - token=settings.YANDEX_DISK_OAUTH_TOKEN, - ) as client: - return func(client, *args, **kwargs) - except PathNotFoundError: - error_message = "Файл не найден на Yandex Disk." - logger.exception(error_message) - raise PathNotFoundError(msg=error_message) - except ForbiddenError: - error_message = "Не хватает прав, чтобы выполнить запрос." - logger.exception(error_message) - raise ForbiddenError(msg=error_message) - except ResourceIsLockedError: - error_message = "Файл заблокирован другой операцией." - logger.exception(error_message) - raise ResourceIsLockedError(msg=error_message) - - return wrapper - - -@yadisk_client -def check_file_exists_on_disk(client: Client, file_path: str) -> bool: - """Проверить существование файла на Yandex Disk.""" - return client.exists(file_path) - - -@yadisk_client -def download_file_by_link( - client: Client, - video_link: str, - media_data_path: str, -) -> None: - """Скачать файл с Yandex Disk по ссылке.""" - client.download_public(video_link, media_data_path) - - -def check_player_game_exists_on_disk(player_game_file_name: str) -> bool: - """Проверить существование файла с игроком на Yandex Disk.""" - player_game_path_on_disk = os.path.join( - YadiskDirectory.PLAYER_GAMES, - player_game_file_name, - ) - return check_file_exists_on_disk(player_game_path_on_disk) - - -def download_file_by_link_task( - video_link: str, - media_data_path: str, -): - """Задача для скачивания файла с Yandex Disk.""" - if os.path.exists(media_data_path): - logger.info(f"Файл {media_data_path} уже скачан на диске.") - return - download_file_by_link( - video_link=video_link, - media_data_path=media_data_path, - ) diff --git a/adaptive_hockey_federation/games/signals.py b/adaptive_hockey_federation/games/signals.py index 711b8f98..f24f2631 100644 --- a/adaptive_hockey_federation/games/signals.py +++ b/adaptive_hockey_federation/games/signals.py @@ -34,10 +34,6 @@ def create_game_teams(sender, instance, created, **kwargs): # конкретно изменение # номеров игроков. Оно должно происходить до того # как объект игры попадёт в бд. - # TODO отправлять видео на распознавание до изменения номеров игроков - # нелогично. - # if created and instance.video_link: - # send_game_video_to_process(instance.id) @receiver(post_save, sender=GameTeam, dispatch_uid="unique_signal") diff --git a/adaptive_hockey_federation/games/urls.py b/adaptive_hockey_federation/games/urls.py index a5997fb2..e4b0fea3 100644 --- a/adaptive_hockey_federation/games/urls.py +++ b/adaptive_hockey_federation/games/urls.py @@ -26,11 +26,6 @@ views.GamesInfoView.as_view(), name="game_info", ), - path( - "/process/", - views.send_game_video_to_process_view, - name="send_game_video_to_process_view", - ), ] urlpatterns = [ diff --git a/adaptive_hockey_federation/games/views.py b/adaptive_hockey_federation/games/views.py index 730b7a6a..ad502fb3 100644 --- a/adaptive_hockey_federation/games/views.py +++ b/adaptive_hockey_federation/games/views.py @@ -1,15 +1,12 @@ import logging from dataclasses import dataclass -from requests.exceptions import RequestException + from typing import Any -from django.contrib import messages from django.contrib.auth.mixins import ( LoginRequiredMixin, PermissionRequiredMixin, ) -from django.contrib.auth.decorators import login_required from django.db.models.query import QuerySet -from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import get_object_or_404, redirect from django.urls import reverse_lazy from django.views.generic import DetailView @@ -26,11 +23,7 @@ from games.mixins import GameCreateUpdateMixin from games.models import Game, GamePlayer, GameTeam from core.logging import configure_logging -from service.a_hockey_requests import send_request_to_process_video -from service.a_hockey_requests import check_api_health_status -# TODO раскоментировать после добавления celery -# from video_api.tasks import get_player_video_frames -from video_api.serializers import GameFeatureSerializer + configure_logging() logger = logging.getLogger(__name__) @@ -236,67 +229,3 @@ def get_context_data(self, **kwargs): ) context["page_title"] = "Редактирование номеров игроков команды" return context - - -def send_game_video_to_process( - game_id: int, - user_email: str = None, -) -> Message: - """Функция формирует данные для запроса к серверу DS.""" - game = get_object_or_404(Game, id=game_id) - game_data = GameFeatureSerializer(game).data - kwargs = { - "data": game_data, - "user_email": user_email, - } - try: - check_api_health_status() - except RequestException as error: - message = Message( - messages.ERROR, - "Сервис по обработке видео недоступен", - ) - logger.error(f"Ошибка подключения к серверу распознавания: {error}") - return message - - logger.info("Отправляем видео на обработку") - send_request_to_process_video(kwargs["data"]) - message = Message( - messages.INFO, - "Видео отправлено на обработку, ждите оповещение " - "о готовности на электронную почту.", - ) - return message - - -@login_required -def send_game_video_to_process_view( - request: HttpRequest, - **kwargs: int, -) -> HttpResponseRedirect | HttpResponse: - """ - Вью функция дли запуска задачи в celery для отправки видео на обработку. - - Функция обрабатывает запрос с кнопки, добавляет задачу в очередь и - отображает сообщение об успешном выполнении. - """ - game_id = kwargs.get("game_id") - - if not Game.objects.get(pk=game_id).video_link: - message = Message( - messages.WARNING, - "Ссылка на видео с игрой не указана.", - ) - else: - message = send_game_video_to_process( - game_id=game_id, - user_email=request.user.email, - ) - - messages.add_message( - request, - message.level, - message.text, - ) - - return redirect("games:game_info", game_id=game_id) diff --git a/adaptive_hockey_federation/main/controllers/player_views.py b/adaptive_hockey_federation/main/controllers/player_views.py index 4c145a8a..c8670517 100644 --- a/adaptive_hockey_federation/main/controllers/player_views.py +++ b/adaptive_hockey_federation/main/controllers/player_views.py @@ -1,29 +1,20 @@ -import os from typing import Any -from django.conf import settings -from django.contrib import messages from django.contrib.auth.mixins import ( LoginRequiredMixin, PermissionRequiredMixin, ) -from django.contrib.auth.decorators import login_required from django.http import Http404 -from django.shortcuts import get_object_or_404, render, redirect +from django.shortcuts import get_object_or_404, render from django.urls import reverse, reverse_lazy from django.views.generic.detail import DetailView from django.views.generic.edit import CreateView, DeleteView, UpdateView from django.views.generic.list import ListView -from requests.exceptions import RequestException -from core.constants import Directory, FileConstants, PLAYER_GAME_NAME +from core.constants import FileConstants from core.utils import is_uploaded_file_valid -from core.ydisk_utils.utils import ( - download_file_by_link_task, - check_player_game_exists_on_disk, -) -from games.models import Game, GamePlayer, GameDataPlayer +from games.models import Game, GamePlayer from main.controllers.mixins import DiagnosisListMixin from main.controllers.utils import errormessage from main.forms import PlayerForm, PlayerUpdateForm @@ -35,10 +26,7 @@ get_player_fields_personal, get_player_table_data, ) -from service.a_hockey_requests import check_api_health_status from unloads.utils import model_get_queryset -from video_api.tasks import create_player_video, get_player_video_frames -from video_api.serializers import GameFeatureSerializer class PlayersListView( @@ -417,146 +405,6 @@ def get_context_data(self, **kwargs) -> dict[str, Any]: return context -@login_required -def unload_player_game_video(request, **kwargs): - """ - Обрабатывает запрос на получение видео с моментами игрока из игры. - - Функция выполняет следующие шаги: - 1. Получает игрока и игру по идентификаторам. - 2. Формирует путь для сохранения видео игры и обработки моментов с игроком. - 3. Проверяет наличие уже существующего видео с моментами игрока на я.диске: - - Если видео уже существует, возвращает ссылку на его скачивание. - - Если видео отсутствует, проверяет наличие фреймов игрока в бд: - - Если фреймы есть, запускает процесс скачивания видео игры, нарезки - моментов с игроком, загрузки видео на я.диск и отправки ссылки - пользователю. - - Если фреймов нет, запускает полный процесс обработки, включая - получение данных с сервера, скачивание видео игры, нарезку моментов, - загрузку видео и отправку ссылки пользователю. - 4. Отправляет сообщение пользователю с информацией о статусе обработки - видео и ссылки на его скачивание. - 5. Перенаправляет пользователя на страницу с видео игрока. - """ - player_id = kwargs["player_id"] - player = get_object_or_404(Player, pk=player_id) - game = get_object_or_404(Game, pk=kwargs["game_id"]) - game_data = GameFeatureSerializer(game).data - player_game_file_name = PLAYER_GAME_NAME.format( - surname=player.surname, - name=player.name, - patronymic=player.patronymic, - game_name=game.name, - ) - - # Директория для скаченных видео с играми. - games_dir = os.path.join( - settings.MEDIA_ROOT, - Directory.GAMES, - ) - os.makedirs(games_dir, exist_ok=True) - game_path = os.path.join( - games_dir, - f"{game.name}.mp4", - ) - - # Директория для обработанных моментов с игроком. - player_games_dir = os.path.join( - settings.MEDIA_ROOT, - Directory.PLAYER_VIDEO_DIR, - ) - os.makedirs(player_games_dir, exist_ok=True) - player_game_frames_path = os.path.join( - player_games_dir, - player_game_file_name, - ) - - if check_player_game_exists_on_disk(player_game_file_name): - # Проверяем есть ли видео с моментами игрока на я.диске. - - # process_chain = chain( - # TODO реализовать таску по отправке ссылки пользователю - # ) - - message_text = "Ссылка для скачивания видео отправлена на почту." - elif GameDataPlayer.objects.filter(player=player, game=game).exists(): - # Проверяем если ли фреймы с игроком в бд. Если есть, то: - - # process_chain = chain( - # download_file_by_link_task.si(game.video_link, game_path).set( - # queue="download_game_video_queue", - # ), - # create_player_video.si( - # game_path, - # player_game_frames_path, - # player.id, - # game.id, - # ).set(queue="slice_player_video_queue"), - # # TODO реализовать таску по загрузке видео с игроком на Я.диск - # # TODO реализовать таску по отправке ссылки пользователю - # ) - - message_text = ( - "Видео находится в обработке. " - "Ссылка для скачивания видео будет отправлена на почту." - ) - else: - # Если нет ни видео, не фреймов, то запускаем полный цикл тасков. - # # TODO реализовать таску по загрузке видео с игроком на Я.диск - # # TODO реализовать таску по отправке ссылки пользователю - try: - download_file_by_link_task(game.video_link, game_path) - except RequestException as e: - messages.add_message( - request, - messages.ERROR, - f"Произошла ошибка при скачивании видео: {e}", - ) - return redirect( - "main:player_id_games_video", - pk=player_id, - ) - try: - check_api_health_status() - except RequestException: - messages.add_message( - request, - messages.ERROR, - "Сервис по обработке видео недоступен", - - ) - return redirect( - "main:player_id_games_video", - pk=player_id, - ) - - frames = get_player_video_frames(game_data) - player_frames = [ - frame["frames"] for frame in frames if frame["number" - ] == player.number] - - create_player_video(input_file=game_path, - output_file=player_game_frames_path, - frames=player_frames[0]) - message_text = ( - "Видео находится в обработке. " - "Ссылка для скачивания видео будет отправлена на почту." - ) - - messages.add_message( - request, - messages.INFO, - message_text, - ) - - # TODO видео будет автоматически загрузаться пользователю по готовности. - # Возможно нужно ресерчить тему WebSockets, SSE - return redirect( - "main:player_id_games_video", - pk=player_id, - ) - - def player_id_deleted(request): """View для отображения информации об успешном удалении игрока.""" return render(request, "main/player_id/player_id_deleted.html") diff --git a/adaptive_hockey_federation/main/urls.py b/adaptive_hockey_federation/main/urls.py index 433c285c..2b72ea5a 100644 --- a/adaptive_hockey_federation/main/urls.py +++ b/adaptive_hockey_federation/main/urls.py @@ -45,11 +45,6 @@ player_views.player_id_deleted, name="player_id_deleted", ), - path( - "/unload/game_video//", - player_views.unload_player_game_video, - name="unload_player_video", - ), ] teams_urlpatterns = [ diff --git a/adaptive_hockey_federation/service/a_hockey_requests.py b/adaptive_hockey_federation/service/a_hockey_requests.py index 3a040f13..c48a5594 100644 --- a/adaptive_hockey_federation/service/a_hockey_requests.py +++ b/adaptive_hockey_federation/service/a_hockey_requests.py @@ -1,5 +1,4 @@ import logging -from typing import Any from urllib.parse import urljoin import requests @@ -24,29 +23,3 @@ def check_api_health_status() -> None: raise RequestException( f"Сервис по обработке видео недоступен: {error}", ) from error - - -def send_request_to_process_video( - data: dict[str, Any], -) -> dict[str, Any]: - """ - Отправка запроса к эндпоинту /process для обработки видео. - - :param data: Словарь с данными для обработки видео. - :returns: Результат обработки видео. - :raises RequestException: Если возникла ошибка при обработке видео. - """ - logger.info("Отправка запроса к серверу DS.") - try: - response = requests.post( - urljoin(settings.PROCESSING_SERVICE_BASE_URL, "/process"), - json=data, - timeout=(0.5, None), - ) - return response.json() - except RequestException as error: - logger.error(f"Ошибка подключения к серверу распознавания: {error}") - return { - "message": "Возникла ошибка при попытке обработать видео: " - "Ошибка подключения к серверу распознавания.", - } diff --git a/adaptive_hockey_federation/templates/main/games/game_detail.html b/adaptive_hockey_federation/templates/main/games/game_detail.html index c85b55f7..4eb755bf 100644 --- a/adaptive_hockey_federation/templates/main/games/game_detail.html +++ b/adaptive_hockey_federation/templates/main/games/game_detail.html @@ -2,96 +2,89 @@ {% load user_filters %} {% block title %} - {{ page_title }} +{{ page_title }} {% endblock title %} {% block content %} - {% include 'base/messages.html' %} -

Детали игры

+{% include 'base/messages.html' %} +

Детали игры

-
-

Название: {{ object.name }}

-
Дата: {{ object.date }}
-
+
+

Название: {{ object.name }}

+
Дата: {{ object.date }}
+
-

Ссылка на видео: {{ object.video_link }}

-

Ссылка на соревнование: {{ object.competition }}

+

Ссылка на видео: {{ object.video_link }}

+

Ссылка на соревнование: {{ object.competition + }}

-
-
-
- {% if teams|length > 0 %} -

{{ teams.0.name }}

- - - - - - - - - {% for player in teams.0.players %} - - - - - {% empty %} - - - - {% endfor %} - -
Имя игрокаНомер игрока
{{ player.last_name }} {{ player.name }}{{ player.number }}
В этой команде нет игроков.
- - Редактировать номера игроков - - {% else %} -

К сожалению, информация о первой команде недоступна.

- {% endif %} -
+
+
+
+ {% if teams|length > 0 %} +

{{ teams.0.name }}

+ + + + + + + + + {% for player in teams.0.players %} + + + + + {% empty %} + + + + {% endfor %} + +
Имя игрокаНомер игрока
{{ player.last_name }} {{ player.name }}{{ player.number }}
В этой команде нет игроков.
+ + Редактировать номера игроков + + {% else %} +

К сожалению, информация о первой команде недоступна.

+ {% endif %} +
-
- {% if teams|length > 1 %} -

{{ teams.1.name }}

- - - - - - - - - {% for player in teams.1.players %} - - - - - {% empty %} - - - - {% endfor %} - -
Имя игрокаНомер игрока
{{ player.last_name }} {{ player.name }}{{ player.number }}
В этой команде нет игроков.
- - Редактировать номера игроков - - {% else %} -

К сожалению, информация о второй команде недоступна.

- {% endif %} -
-
-
-
- - Отправить видео на распознание - +
+ {% if teams|length > 1 %} +

{{ teams.1.name }}

+ + + + + + + + + {% for player in teams.1.players %} + + + + + {% empty %} + + + + {% endfor %} + +
Имя игрокаНомер игрока
{{ player.last_name }} {{ player.name }}{{ player.number }}
В этой команде нет игроков.
+ + Редактировать номера игроков + + {% else %} +

К сожалению, информация о второй команде недоступна.

+ {% endif %} +
+
{% endblock content %} diff --git a/adaptive_hockey_federation/video_api/__init__.py b/adaptive_hockey_federation/video_api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/adaptive_hockey_federation/video_api/apps.py b/adaptive_hockey_federation/video_api/apps.py deleted file mode 100644 index 3afb9217..00000000 --- a/adaptive_hockey_federation/video_api/apps.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.apps import AppConfig - - -class VideoApiConfig(AppConfig): - """Конфигурация приложения Video API.""" - - default_auto_field = "django.db.models.BigAutoField" - name = "video_api" diff --git a/adaptive_hockey_federation/video_api/permissions.py b/adaptive_hockey_federation/video_api/permissions.py deleted file mode 100644 index d1f93e53..00000000 --- a/adaptive_hockey_federation/video_api/permissions.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.conf import settings -from rest_framework.permissions import BasePermission - - -class HasAPIDocsKey(BasePermission): - """Кастомный пермишен для API ключа.""" - - def has_permission(self, request, view): - """Проверка наличия и соответствия API ключа в заголовках запроса.""" - api_key = request.headers.get("X-API-KEY") - return api_key == settings.API_DOCS_KEY diff --git a/adaptive_hockey_federation/video_api/serializers.py b/adaptive_hockey_federation/video_api/serializers.py deleted file mode 100644 index cf40a9ac..00000000 --- a/adaptive_hockey_federation/video_api/serializers.py +++ /dev/null @@ -1,119 +0,0 @@ -from django.conf import settings -from rest_framework import serializers - -from games.models import Game, GamePlayer -from main.models import Player - - -class GameFeatureSerializer(serializers.ModelSerializer): - """Сериализатор подготовки данных для отправки к DS.""" - - game_id = serializers.IntegerField( - source="id", - ) - game_link = serializers.CharField( - source="video_link", - read_only=True, - ) - team_ids = serializers.SerializerMethodField() - player_ids = serializers.SerializerMethodField() - player_numbers = serializers.SerializerMethodField() - token = serializers.SerializerMethodField() - - class Meta: - model = Game - fields = [ - "game_id", - "game_link", - "player_ids", - "player_numbers", - "team_ids", - "token", - ] - - def sort_player_by_team( - self, - obj: Game, - field_name: str, - ) -> list[list[int]]: - """ - Метод для сортировки игроков по командам. - - Сортировка в соответствии с порядком команд в поле team_ids. - """ - game_players = GamePlayer.objects.filter(game_team__game=obj) - team_players: list[tuple[int, int]] = list( - game_players.values_list("game_team_id", field_name), - ) - teams: dict[int, list[int]] = { - team_id: [] for team_id in self.get_team_ids(obj) - } - for team_id, field in team_players: - teams[team_id].append(field) - return list(teams.values()) - - def get_team_ids(self, obj): - """Метод получения ID команд.""" - return list(obj.game_teams.values_list("gameteam_id", flat=True)) - - def get_player_ids(self, obj): - """Метод получения ID игроков.""" - return self.sort_player_by_team(obj, "id") - - def get_player_numbers(self, obj): - """Метод получения номеров игроков.""" - return self.sort_player_by_team(obj, "number") - - def get_token(self, obj): - """Метод токена.""" - return settings.YANDEX_DISK_OAUTH_TOKEN - - -# TODO возможно верный сериализатор DS. Уточнить структуру ответа DS -class TrackingSerializer(serializers.Serializer): - """Сериализатор, обрабатывающий tracking с фреймами.""" - - player_id = serializers.PrimaryKeyRelatedField( - queryset=Player.objects.all(), - ) - team_id = serializers.IntegerField() - frames = serializers.ListField( - child=serializers.IntegerField(), - ) - boxes = serializers.ListField( - child=serializers.ListField( - child=serializers.IntegerField(), - ), - ) - time = serializers.ListField( - child=serializers.CharField(max_length=50), - ) - predicted_number = serializers.SerializerMethodField() - - def get_predicted_number(self, obj): - """Данные predicted_number могут быть как str так и int.""" - return obj.predicted_number - - -# TODO возможно верный сериализатор DS. Уточнить структуру ответа DS -class GameDataPlayerSerializer(serializers.Serializer): - """Сериалатор для маршалинга ответа от сервиса дс-ов.""" - - game_id = serializers.PrimaryKeyRelatedField( - queryset=Game.objects.all(), - ) - tracking = TrackingSerializer( - many=True, - ) - - -# TODO уточнить структуру ответа DS -class GameDataPlayerSerializerMock(serializers.Serializer): - """Сериализатор заглушка ответа DS.""" - - number = serializers.IntegerField() - team = serializers.IntegerField() - counter = serializers.IntegerField() - frames = serializers.ListField( - child=serializers.IntegerField(), - ) diff --git a/adaptive_hockey_federation/video_api/tasks.py b/adaptive_hockey_federation/video_api/tasks.py deleted file mode 100644 index 5cbe147f..00000000 --- a/adaptive_hockey_federation/video_api/tasks.py +++ /dev/null @@ -1,150 +0,0 @@ -# import json -import logging -import os - -# from django.db import transaction - -# from games.models import Game, GameDataPlayer, GamePlayer -# from main.models import Player -from service.a_hockey_requests import send_request_to_process_video -from service.video_processing import slicing_video_with_player_frames -# from users.utilits.send_mails import send_info_mail -# from .serializers import GameDataPlayerSerializerMock - - -logger = logging.getLogger(__name__) - - -def get_player_video_frames(game_data): - """Таск для запуска обработки видео.""" - logger.info("Добавлен новый объект игры, запускаем воркер") - return send_request_to_process_video(game_data) - - -def create_player_video( - *args, - **kwargs, -): - """Таск для нарезки видео с моментами игрока.""" - # Мок реализация фреймов для нарезки видео с моментами игрока. - # Пока подставляются тестовые фреймы. - # TODO удалить мок реализацию, как в бд появятся фреймы по игрокам. - - input_file = kwargs["input_file"] - output_file = kwargs["output_file"] - # player = kwargs["player"] - # game = kwargs["game"] - frames = kwargs["frames"] - if os.path.exists(output_file): - return - - slicing_video_with_player_frames(input_file, output_file, frames) - return f"Видео обработано. {args}" - -# TODO раскомментировать после добавления celery -# def bulk_create_gamedataplayer_objects(sender=None, **kwargs): -# """ -# Сохранение параметров видео игроков от сервера DS. -# Вызов таски нарезки видео. -# """ -# result = kwargs.get("result") -# task_params = sender.request.kwargs["data"] -# user_email = sender.request.kwargs["user_email"] - -# # TODO уточнить структуру ответа DS -# serializer = GameDataPlayerSerializerMock(data=result, many=True) -# if serializer.is_valid(): -# object_data = serializer.validated_data -# game = Game.objects.get(pk=task_params["game_id"]) -# with transaction.atomic(): -# for track in object_data: -# try: -# game_player = GamePlayer.objects.get( -# game_team__game=game, -# game_team_id=track["team"], -# number=track["number"], -# ) -# except GamePlayer.DoesNotExist: -# logger.warning( -# f"Игрок с номером {track['number']} " -# f"команды {track['team']} " -# f"в игре {task_params['game_id']} не найден.", -# ) -# continue -# except GamePlayer.MultipleObjectsReturned: -# logger.warning( -# f"В команде {track['team']} " -# f"несколько игроков с номером {track['number']} " -# f"участвовало в игре {task_params['game_id']}.", -# ) -# continue -# player = Player.objects.get(pk=game_player.id) -# # TODO возможно следует использовать bulk_create -# GameDataPlayer.objects.update_or_create( -# player=player, -# game=game, -# defaults={"data": json.dumps(track)}, -# ) -# logger.info( -# ( -# f"Cоздаем видео для игрока " -# f"{player.get_name_and_position()}" -# ), -# ) -# # TODO в args передают аргументы -# # нужные для нарезки видео с игроком -# # create_player_video( -# # TODO заменить название исходного файла -# # видео с игрой нужно скачать -# # ссылка на видео с игрой task_params["game_link"] -# input_file = "input_file.mp4" -# output_file = ( -# f"video_game_{task_params['game_id']}_" -# f"player_{game_player.id}.mp4" -# ) -# create_player_video.apply_async( -# args=["Обработка с низким приоритетом"], -# kwargs={ -# "input_file": input_file, -# "output_file": output_file, -# "player": str(player), -# "game": game.name, -# "user_email": user_email, -# "frames": track["frames"], -# "priority": 255, -# }, -# ) -# else: -# logger.error(serializer.errors) - -# TODO раскомментировать после добавления celery -# def on_pool_process_init(**kwargs): -# # Что бы отрабатывал сигнал task_success -# # https://github.com/celery/celery/issues/2343 -# # https://django.fun/docs/celery/5.1/userguide/signals/#worker-process-init -# task_success.connect( -# bulk_create_gamedataplayer_objects, -# sender=current_app.tasks[get_player_video_frames.name], -# ) - - -# TODO раскомментировать после добавления celery -# def send_success_mail(sender=None, **kwargs): -# """Вызывает функцию отправки письма о готовности видео с игроком.""" -# player = sender.request.kwargs["player"] -# game = sender.request.kwargs["game"] -# user_email = sender.request.kwargs["user_email"] -# send_info_mail( -# "Обработка видео завершена", -# f'Завершена обработка видео игрока {player} в игре "{game}".', -# user_email, -# ) - - -# TODO раскомментировать после добавления celery -# def mail_success_video_process(**kwargs): -# """Обработка сигнала task_success таски create_player_video.""" -# task_success.connect( -# send_success_mail, -# sender=current_app.tasks[create_player_video.name], -# ) From 74e89cf3ef6ee61d4a2f62fe906d5c27ac585062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD?= <128288828+InKLaR1TY@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:46:39 +0300 Subject: [PATCH 4/4] Remove parser and rewrite command fill-db (#554) --- .pre-commit-config.yaml | 4 +- .../core/management/commands/fill-db.py | 155 ++++-- adaptive_hockey_federation/parser/__init__.py | 0 .../parser/docx_parser.py | 517 ------------------ .../parser/exception.py | 2 - .../parser/importing_db.py | 261 --------- adaptive_hockey_federation/parser/parser.py | 108 ---- .../parser/user_card.py | 46 -- .../parser/xlsx_parser.py | 31 -- pyproject.toml | 4 - requirements/develop.txt | 23 +- requirements/production.txt | 18 +- 12 files changed, 111 insertions(+), 1058 deletions(-) delete mode 100644 adaptive_hockey_federation/parser/__init__.py delete mode 100644 adaptive_hockey_federation/parser/docx_parser.py delete mode 100644 adaptive_hockey_federation/parser/exception.py delete mode 100644 adaptive_hockey_federation/parser/importing_db.py delete mode 100644 adaptive_hockey_federation/parser/parser.py delete mode 100644 adaptive_hockey_federation/parser/user_card.py delete mode 100644 adaptive_hockey_federation/parser/xlsx_parser.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdcc6f2c..b60491fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-docstring-first - id: check-merge-conflict @@ -8,7 +8,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.5 + rev: v0.6.8 hooks: - id: ruff exclude: migrations/|config/|tests/|.*settings(\.py|/)? diff --git a/adaptive_hockey_federation/core/management/commands/fill-db.py b/adaptive_hockey_federation/core/management/commands/fill-db.py index 9b5cd617..b5ba070b 100644 --- a/adaptive_hockey_federation/core/management/commands/fill-db.py +++ b/adaptive_hockey_federation/core/management/commands/fill-db.py @@ -1,18 +1,15 @@ +import json + +from django.apps import apps from django.core.management.base import BaseCommand +from django.db import connection, transaction +from main.models import Diagnosis from adaptive_hockey_federation.core.config.dev_settings import ( FILE_MODEL_MAP, FIXSTURES_DIR, - FIXSTURES_FILE, -) -from adaptive_hockey_federation.parser.importing_db import ( - clear_data_db, - importing_parser_data_db, - importing_real_data_db, ) -DB_MESSAGE = "Данные успешно добавлены!" - class Command(BaseCommand): """Класс для парсинга данных и их записи в БД.""" @@ -20,13 +17,7 @@ class Command(BaseCommand): help = "Запуск парсера офисных документов, и запись их в БД." def add_arguments(self, parser): - """Добавляет новые аргументы для командной строки.""" - parser.add_argument( - "-p", - "--parser", - action="store_true", - help="Запуск парсера документов", - ) + """Аргументы.""" parser.add_argument( "-f", "--fixtures", @@ -34,50 +25,116 @@ def add_arguments(self, parser): help="Фикстуры с реальными данными для таблиц.", ) - def load_data(self) -> None: - """Загрузка распарсенных данных.""" - importing_parser_data_db(FIXSTURES_FILE) - return None - - def load_real_data(self) -> None: + @transaction.atomic + def load_real_data(self) -> None: # noqa: C901 """Загрузка реальных данных из JSON.""" - for key in FILE_MODEL_MAP.items(): - file_name = key[0] + ".json" - if "main_" in key[0]: + with connection.cursor() as cursor: + for table_name in FILE_MODEL_MAP.keys(): try: - clear_data_db(file_name) + cursor.execute(f"TRUNCATE TABLE {table_name} CASCADE") except Exception as e: - self.stdout.write( - self.style.ERROR( - f"Ошибка удаления данных {e} -> " f"{file_name}", + return self.stdout.write( + self.style.WARNING( + f"Не удалось очистить таблицу {table_name}: {str(e)}", # noqa: E501 ), ) + items = list(FILE_MODEL_MAP.items()) items.reverse() - for key in items: - file_name = key[0] + ".json" - if "main_" in key[0]: - try: - importing_real_data_db(FIXSTURES_DIR, file_name) - self.stdout.write( - self.style.SUCCESS( - f"Фикстуры с файла {file_name} вставлены " - "в таблицы!", - ), - ) - except Exception as e: - return self.stdout.write( - self.style.ERROR_OUTPUT( - f"Ошибка вставки данных {e} -> " f"{file_name}", - ), - ) - return None + for table_name, model_class in items: + file_path = FIXSTURES_DIR / f"{table_name}.json" + app_label, model_name = table_name.split("_", 1) + model = apps.get_model(app_label, model_name) + try: + with open(file_path, "r", encoding="utf-8") as file: + data = json.load(file) + + for item in data: + if table_name == "main_team": + team_data = { + "id": item["id"], + "name": item["name"], + "city_id": item["city_id"], + "discipline_name_id": item["discipline_name_id"], + "curator_id": 1, + } + instance = model(**team_data) + elif table_name == "main_player": + disciplines = self.get_disciplines() + diagnosis = None + if item.get("diagnosis_id"): + try: + diagnosis = Diagnosis.objects.get( + pk=item["diagnosis_id"], + ) + except Diagnosis.DoesNotExist: + print( + f"Диагноз с id {item.get('diagnosis_id')} отсутсвует.", # noqa: E501 + ) + player_data = { + "id": item["id"], + "surname": item["surname"], + "name": item["name"], + "patronymic": item["patronymic"], + "birthday": item["birthday"], + "gender": item["gender"], + "level_revision": item["level_revision"], + "position": item["position"], + "number": item["number"], + "is_captain": item["is_captain"], + "is_assistent": item["is_assistent"], + "identity_document": item["identity_document"], + "diagnosis": diagnosis, + "diagnosis_id": item["diagnosis_id"], + "discipline_name_id": disciplines[ + item["discipline_id"] + ]["discipline_name_id"], + "discipline_level_id": disciplines[ + item["discipline_id"] + ]["discipline_level_id"], + } + instance = model(**player_data) + else: + instance = model(**item) + instance.save() + + self.stdout.write( + self.style.SUCCESS( + f"Данные из {table_name}.json успешно загружены в модель {model_class}", # noqa: E501 + ), + ) + + except FileNotFoundError: + self.stdout.write( + self.style.WARNING(f"Файл {table_name}.json не найден"), + ) + except Exception as e: + self.stdout.write( + self.style.ERROR( + f"Ошибка при загрузке данных из {table_name}.json: {str(e)}", # noqa: E501 + ), + ) def handle(self, *args, **options): """Запись данных в БД.""" - parser = options.get("parser") fixtures = options.get("fixtures") if fixtures: self.load_real_data() - if parser: - self.load_data() + + def get_disciplines(self) -> dict: + """Получение диспциплин.""" + with open( + FIXSTURES_DIR / "main_discipline.json", + "r", + encoding="utf-8", + ) as file: + data = json.load(file) + disciplines = { + None: {"discipline_level_id": None, "discipline_name_id": None}, + } + for item in data: + disciplines[item["id"]] = { + "discipline_level_id": item["discipline_level_id"], + "discipline_name_id": item["discipline_name_id"], + } + return disciplines diff --git a/adaptive_hockey_federation/parser/__init__.py b/adaptive_hockey_federation/parser/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/adaptive_hockey_federation/parser/docx_parser.py b/adaptive_hockey_federation/parser/docx_parser.py deleted file mode 100644 index 24f583be..00000000 --- a/adaptive_hockey_federation/parser/docx_parser.py +++ /dev/null @@ -1,517 +0,0 @@ -import re -from datetime import date -from typing import Optional - -import docx # type: ignore - -from adaptive_hockey_federation.parser.user_card import BaseUserInfo - -NAME = "[И|и][М|м][Я|я]|Ф.И.О." -SURNAME = "[Ф|ф][А|а][М|м][И|и][Л|л][И|и][Я|я]|Ф.И.О." -PATRONYMIC = "[О|о][Т|т]?[Ч|ч][Е|е][С|с][Т|т][В|в][О|о]|Ф.И.О." -DATE_OF_BIRTH = "[Д|д][А|а][Т|т][А|а] [Р|р][О|о].+" -TEAM = "[К|к][О|о][М|м][А|а][Н|н][Д|д][А|а]" -PLAYER_NUMBER = "[И|и][Г|г][Р|р][О|о][В|в][О|о][Й|й]" -POSITION = "[П|п][О|о][З|з][И|и][Ц|ц][И|и][Я|я]|Должность" -NUMERIC_STATUS = "[Ч|ч].+[С|с][Т|т].+" -PLAYER_CLASS = "[К|к][Л|л][А|а][С|с][С|с]" -PASSPORT = "[П|а][С|с][П|п][О|о][Р|р][Т|т]" -ASSISTENT = ["А", "а", "(А)", "(а)", "Ассистент", "ассистент"] -CAPTAIN = ["К", "к", "(К)", "(к)", "Капитан", "капитан"] -DISCIPLINE_LEVEL = "без ограничений" - - -def read_file_columns(file: docx) -> list[docx]: - """Функция находит таблицы в файле и возвращает список объектов - docx с данными каждого столбца. - """ - return [ - column for table in file.tables for index, column in enumerate(table.columns) - ] - - -def read_file_text(file: docx) -> list[str]: - """Функция находит текстовые данные в файле и возвращает список объектов - docx с найденными данными. - """ - return [run.text for paragraph in file.paragraphs for run in paragraph.runs] - - -def get_counter_for_columns_parser(columns: list[docx]) -> int: - count = 0 - for column in columns: - for index, cell in enumerate(column.cells): - if re.search(r"п/п", cell.text): - for cell in column.cells[index + 1 :]: - if cell.text and len(cell.text) < 4: - count += 1 - else: - break - else: - if count > 0: - break - return count - - -def columns_parser( - columns: list[docx], - regular_expression: str, -) -> list[Optional[str]]: - """Функция находит столбец по названию и списком выводит содержимое - каждой ячейки этого столбца. - """ - output = [ - text if text else None - for column in columns - if re.search(regular_expression, list(cell.text for cell in column.cells)[0]) - for text in list(cell.text for cell in column.cells)[1:] - ] - if not output: - count = get_counter_for_columns_parser(columns) - for column in columns: - for index, cell in enumerate(column.cells): - if re.search(regular_expression, cell.text): - for cell in column.cells[index + 1 : index + 1 + count]: - output.append(cell.text) - return output - - -def find_names(columns: list[docx], regular_expression: str) -> list[str]: - """Функция парсит в искомом столбце имена. Опирается на шаблон ФИО - (имя идет после фамилии на втором месте). - """ - names_list = columns_parser(columns, regular_expression) - return [name.split()[1].rstrip() for name in names_list if name] - - -def find_surnames(columns: list[docx], regular_expression: str) -> list[str]: - """Функция парсит в искомом столбце фамилии. Опирается на шаблон ФИО - (фамилия идет на первом месте). - """ - surnames_list = columns_parser(columns, regular_expression) - return [surname.split()[0].rstrip() for surname in surnames_list if surname] - - -def find_patronymics( - columns: list[docx], - regular_expression: str, -) -> list[str]: - """Функция парсит в искомом столбце отчества. Опирается на шаблон ФИО - (отчество идет на последнем месте). - """ - patronymics_list = columns_parser(columns, regular_expression) - return [ - ( - patronymic.replace("/", " ").split()[2].rstrip().rstrip(",") - if patronymic and len(patronymic.split()) > 2 - else "Отчество отсутствует" - ) - for patronymic in patronymics_list - ] - - -def find_dates_of_birth( - columns: list[docx], - regular_expression: str, -) -> list[date]: - """Функция парсит в искомом столбце дату рождения - и опирается на шаблон дд.мм.гггг. - """ - dates_of_birth_list = columns_parser(columns, regular_expression) - dates_of_birth_list_clear = [] - for date_of_birth in dates_of_birth_list: - if date_of_birth: - try: - for day, month, year in [re.sub(r"\D", " ", date_of_birth).split()]: - if len(year) == 2: - if int(year) > 23: - year = "19" + year - else: - year = "20" + year - dates_of_birth_list_clear.append( - date(int(year), int(month), int(day)) - ) - except ValueError or IndexError: - dates_of_birth_list_clear.append(date(1900, 1, 1)) - else: - dates_of_birth_list_clear.append(date(1900, 1, 1)) - - return dates_of_birth_list_clear - - -def find_team( - text: list[str], - columns: list[docx], - regular_expression: str, -) -> str: - """Функция парсит название команды.""" - text_clear = " ".join(text) - text_clear = re.sub( - r"\W+|_+|ХК|СХК|ДЮСХК|Хоккейный клуб|по незрячему хоккею" - "|по специальному хоккею|Спец хоккей|по специальному|по следж-хоккею", - " ", - text_clear, - ).split() - try: - return [ - ( - "Молния Прикамья" - if text_clear[index + 2] == "Прикамья" - else ( - "Ак Барс" - if text_clear[index + 1] == "Ак" - else ( - "Снежные Барсы" - if text_clear[index + 1] == "Снежные" - else ( - "Хоккей Для Детей" - if text_clear[index + 1] == "Хоккей" - else ( - "Дети-Икс" - if text_clear[index + 1] == "Дети" - else ( - "СКА-Стрела" - if text_clear[index + 1] == "СКА" - else ( - "Сборная Новосибирской области" - if text_clear[index + 2] == "Новосибирской" - else ( - "Атал" - if text_clear[index + 3] == "Атал" - else ( - "Крылья Мечты" - if text_clear[index + 2] == "мечты" - else ( - "Огни Магнитки" - if text_clear[index + 1] == "Огни" - else ( - "Энергия Жизни Краснодар" - if text_clear[index + 3] - == "Краснодар" - else ( - "Энергия Жизни Сочи" - if text_clear[index + 4] - == "Сочи" - else ( - "Динамо-Москва" - if text_clear[index + 1] - == "Динамо" - else ( - "Крылья Советов" - if text_clear[ - index + 2 - ] - == "Советов" - else ( - "Красная Ракета" - if text_clear[ - index + 2 - ] - == "Ракета" - else ( - "Красная Молния" - if text_clear[ - index - + 2 - ] - == "молния" - else ( - "Сахалинские Львята" - if text_clear[ - index - + 1 - ] - == "Сахалинские" - else ( - "Мамонтята Югры" - if text_clear[ - index - + 1 - ] - == "Мамонтята" - else ( - "Уральские Волки" - if text_clear[ - index - + 1 - ] - == "Уральские" - else ( - "Нет названия команды" - if text_clear[ - index - + 1 - ] - == "Всего" - else text_clear[ - index - + 1 - ].capitalize() - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - ) - for index, txt in enumerate(text_clear) - if re.search(regular_expression, txt) - ][0] - except IndexError: - for column in columns: - for cell in column.cells: - if re.search(regular_expression, cell.text): - txt = re.sub(r"\W", " ", cell.text) - return txt.split()[1].capitalize() - - return "Название команды не найдено" - - -def find_players_number( - columns: list[docx], - regular_expression: str, -) -> list[int]: - """Функция парсит в искомом столбце номер игрока.""" - players_number_list = columns_parser(columns, regular_expression) - players_number_list_clear = [] - for player_number in players_number_list: - if player_number: - try: - players_number_list_clear.append( - int(re.sub(r"\D", "", player_number)[:2]) - ) - except ValueError: - players_number_list_clear.append(0) - else: - players_number_list_clear.append(0) - - return players_number_list_clear - - -def find_positions(columns: list[docx], regular_expression: str) -> list[str]: - """Функция парсит в искомом столбце позицию игрока на поле.""" - positions_list = columns_parser(columns, regular_expression) - return [ - ( - "нападающий" - if re.search( - r"^н|^Н|^H|^Нп|^нл|^нп|^цн|^лн|^Нап|^№|^А,|^К,", position.lstrip() - ) - else ( - "защитник" - if re.search(r"^з|^З|^Зщ|^Защ", position.lstrip()) - else ( - "вратарь" - if re.search(r"^Вр|^В|^вр", position.lstrip()) - else ( - "Позиция записана неверно" - if not re.sub(r"\n|\(.+|\d", "", position) - else re.sub(r"\n|\(.+|\d|Капитан", "", position) - .lower() - .rstrip() - .replace(",", "") - .lstrip() - ) - ) - ) - ) - for position in positions_list - if position - ] - - -def find_numeric_statuses(file: docx) -> list[list[str]]: - numeric_statuses_list = [] - for table in file.tables: - for row in table.rows: - txt = row.cells[1].text.title() - txt = re.sub(r"\W|Коляс.+|Здоровый", " ", txt) - if len(txt.split()) <= 4: - try: - numeric_status = row.cells[4].text - numeric_status = re.sub(r"\D", "", numeric_status) - if numeric_status: - if len(txt.split()) == 2: - txt += " Отчество отсутствует" - numeric_statuses_list.append(txt.split()[:3] + [numeric_status]) - except IndexError: - pass - - return numeric_statuses_list - - -def find_passport(columns: list[docx], regular_expression: str) -> list[str]: - """Функция парсит в искомом столбце ПД.""" - identity_list = columns_parser(columns, regular_expression) - return [ - " ".join(identity_data.split()) - for identity_data in identity_list - if identity_data - ] - - -def find_players_is_captain(columns: list[docx], regular_expression: str) -> list[bool]: - """Функция парсит в искомом столбце капитанов.""" - is_captain_list = [] - for is_captain in columns_parser(columns, regular_expression): - for i in CAPTAIN: - if is_captain and i in is_captain.strip(): - try: - is_captain_list.append(True) - except ValueError: - is_captain_list.append(False) - else: - is_captain_list.append(False) - return is_captain_list - - -def find_players_is_assistant( - columns: list[docx], - regular_expression: str, -) -> list[bool]: - """Функция парсит в искомом столбце асситсента.""" - is_assistant_list = [] - for is_assistant in columns_parser(columns, regular_expression): - for i in ASSISTENT: - if is_assistant and i in is_assistant.strip(): - try: - is_assistant_list.append(True) - except ValueError: - is_assistant_list.append(False) - else: - is_assistant_list.append(False) - return is_assistant_list - - -def find_discipline_level( - columns: list[docx], - regular_expression: str, -) -> list[str]: - """Функция парсит в искомом столбце класс/статус.""" - discipline_level_list = [] - for discipline_level in columns_parser(columns, regular_expression): - if discipline_level: - try: - discipline_level = discipline_level.replace("Класс ", "").replace( - "класс ", "" - ) - discipline_level = re.sub( - r"без ограничений|Не имеет ограничений по здоровью" - "|Без ограничений по здоровью" - "|4Без ограничений" - "|Без ограничений\n4|без ограничений по здоровью 2" - "|игрок без ограничений по здоровью" - "|(без ограничений по здоровью) 3" - "|без ограничений 2|Не имеет ограничений по здоровью", - "б\\к", - discipline_level, - ) - if discipline_level != "б\\к": - discipline_level = ( - discipline_level.replace("А", "A") - .replace("С", "C") - .replace("Б", "B") - .replace("б", "B") - ) - discipline_level_list.append(discipline_level) - except ValueError: - discipline_level_list.append("") - else: - discipline_level_list.append("") - return discipline_level_list - - -def numeric_status_check( - name: str, - surname: str, - patronymics: str, - statuses: list[list[str]], -) -> Optional[int]: - for status in statuses: - if surname == status[0]: - if name == status[1]: - if patronymics.split()[0] == status[2]: - return int(status[3]) - - return None - - -def length_list(name: list, len_name: int) -> None: - if len(name) != len_name: - for _ in range(len_name - len(name)): - name.append(None) - return None - - -def docx_parser(path: str, numeric_statuses: list[list[str]]) -> list[BaseUserInfo]: - """Функция собирает все данные об игроке - и передает их в dataclass. - """ - file = docx.Document(path) - columns_from_file = read_file_columns(file) - text_from_file = read_file_text(file) - names = find_names(columns_from_file, NAME) - surnames = find_surnames(columns_from_file, SURNAME) - patronymics = find_patronymics(columns_from_file, PATRONYMIC) - dates_of_birth = find_dates_of_birth( - columns_from_file, - DATE_OF_BIRTH, - ) - team = find_team(text_from_file, columns_from_file, TEAM) - players_number = find_players_number(columns_from_file, PLAYER_NUMBER) - positions = find_positions(columns_from_file, POSITION) - passport = find_passport(columns_from_file, PASSPORT) - is_assistents = find_players_is_assistant(columns_from_file, PLAYER_NUMBER) - is_assistents_alt = find_players_is_assistant(columns_from_file, POSITION) - is_captains = find_players_is_captain(columns_from_file, PLAYER_NUMBER) - is_captains_alt = find_players_is_captain(columns_from_file, POSITION) - classification = find_discipline_level(columns_from_file, DISCIPLINE_LEVEL) - - length_list(players_number, len(names)) - length_list(is_assistents, len(names)) - length_list(is_captains, len(names)) - length_list(classification, len(names)) - length_list(passport, len(names)) - length_list(positions, len(names)) - - return [ - BaseUserInfo( - name=names[index], - surname=surnames[index], - date_of_birth=dates_of_birth[index], - team=team, - player_number=players_number[index], - position=positions[index], - numeric_status=numeric_status_check( - names[index], - surnames[index], - patronymics[index], - numeric_statuses, - ), - patronymic=patronymics[index], - passport=passport[index], - is_assistant=( - is_assistents[index] - if is_assistents[index] - else is_assistents_alt[index] - ), - is_captain=( - is_captains[index] if is_captains[index] else is_captains_alt[index] - ), - classification=classification[index], - ).__dict__ - for index in range(len(names)) - ] diff --git a/adaptive_hockey_federation/parser/exception.py b/adaptive_hockey_federation/parser/exception.py deleted file mode 100644 index c55dfaf8..00000000 --- a/adaptive_hockey_federation/parser/exception.py +++ /dev/null @@ -1,2 +0,0 @@ -class ExceptionForFlake8(Exception): - pass diff --git a/adaptive_hockey_federation/parser/importing_db.py b/adaptive_hockey_federation/parser/importing_db.py deleted file mode 100644 index 1effb0c3..00000000 --- a/adaptive_hockey_federation/parser/importing_db.py +++ /dev/null @@ -1,261 +0,0 @@ -import json -import subprocess - -from django.db import connection, transaction -from main import models -from main.models import (DisciplineLevel, DisciplineName, Player, StaffMember, - StaffTeamMember, Team) - -from adaptive_hockey_federation.core.config.dev_settings import ( - FILE_MODEL_MAP, RESOURSES_ROOT) -from adaptive_hockey_federation.parser.user_card import BaseUserInfo -from main import models -from main.models import (Diagnosis, DisciplineLevel, DisciplineName, Player, StaffMember, - StaffTeamMember, Team, Nosology) - -MODELS_ONE_FIELD_NAME = ["main_city", "main_disciplinename", "main_nosology"] - -PLAYER_POSITIONS = [ - "нападающий", - "поплавок", - "вратарь", - "защитник", - "Позиция записана неверно", -] -STAFF_POSITIONS = [ - "тренер", - "координатор", - "пушер", -] - - -def parse_file(file_path: str) -> list[BaseUserInfo]: - with open(file_path, "r", encoding="utf-8") as file: - data = json.load(file) - return data - - -def get_discipline_name(item_name: str): - try: - discipline_name = DisciplineName.objects.get(name=item_name) - except DisciplineName.DoesNotExist: - discipline_name = None - return discipline_name - - -def get_discipline_level(item_name: str): - try: - discipline_level = DisciplineLevel.objects.get(name=item_name) - except DisciplineLevel.DoesNotExist: - discipline_level = None - return discipline_level - - -def create_staff_member(item): - try: - try: - staff_member = StaffMember( - surname=item["surname"], - name=item["name"], - patronymic=item["patronymic"], - ) - staff_member.save() - - staff_team_member = StaffTeamMember( - staff_position=item["position"], - staff_member_id=staff_member.id, - notes=item["date_of_birth"].replace(" 00:00:00", ""), - ) - - staff_team_member.save() - team = Team.objects.get(name=item["team"]) - staff_team_member_id = StaffTeamMember.objects.get( - staff_position__contains="тренер", pk=staff_team_member.id - ) - if team.staff_team_member_id != staff_team_member_id: - team.staff_team_member_id = staff_team_member_id - team.save() - return team - except StaffTeamMember.DoesNotExist: - team = None - except Exception as e: - print(f"Ошибка вставки данных {e} -> {item}") - - -def create_players(item, discipline_name) -> None: - try: - player_model = Player( - surname=item["surname"], - name=item["name"], - patronymic=item["patronymic"], - birthday=item["date_of_birth"].replace(" 00:00:00", ""), - gender="", - level_revision=item["revision"], - position=item["position"], - number=item["player_number"], - is_captain=item["is_captain"], - is_assistent=item["is_assistant"], - identity_document=item["passport"], - discipline_name=discipline_name, - ) - player_model.save() - teams = Team.objects.get(name=item["team"]) - player_model.team.add(teams) - - except Exception as e: - print(f"Ошибка вставки данных {e} -> {item}") - - -# flake8: noqa: C901 -def importing_parser_data_db(FIXSTURES_FILE: str) -> None: - subprocess.getoutput(f"poetry run parser -r -p {RESOURSES_ROOT}") - data = parse_file(FIXSTURES_FILE) - for item in data: - for key in item: - if item[key] is None and key != "player_number": - item[key] = "" - if item[key] is None and key == "player_number": - item[key] = 0 - for i in PLAYER_POSITIONS: - if i in item["position"]: - create_players(item, get_discipline_name(item["classification"])) - for i in STAFF_POSITIONS: - if i in item["position"]: - create_staff_member(item) - - -def clear_data_db(file_name: str) -> None: - key = file_name.replace(".json", "") - models_name = getattr(models, FILE_MODEL_MAP[key]) - models_name.objects.all().delete() - cursor = connection.cursor() - cursor.execute( - f"SELECT setval(pg_get_serial_sequence('{key}', 'id')," - f"coalesce(max(id), 1), max(id) IS NOT null)" - f"FROM {key};" - ) - - -def parse_disciplines(FIXSTURES_DIR) -> dict: - with open(FIXSTURES_DIR / "main_discipline.json", "r", encoding="utf-8") as file: - data = json.load(file) - disciplines = {None: {"discipline_level_id": None, "discipline_name_id": None}} - for item in data: - disciplines[item["id"]] = { - "discipline_level_id": item["discipline_level_id"], - "discipline_name_id": item["discipline_name_id"], - } - return disciplines - - -def importing_real_data_db(FIXSTURES_DIR, file_name: str) -> None: - with open(FIXSTURES_DIR / file_name, "r", encoding="utf-8") as file: - data = json.load(file) - key = file_name.replace(".json", "") - models_name = getattr(models, FILE_MODEL_MAP[key]) - if key == "main_player": - disciplines = parse_disciplines(FIXSTURES_DIR) - max_id = 0 - for item in data: - max_id = max(max_id, item["id"]) - try: - if key in MODELS_ONE_FIELD_NAME: - model_ins = models_name( - id=item["id"], - name=item["name"] - ) - model_ins.save() - if key == "main_staffmember": - model_ins = models_name( - id=item["id"], - surname=item["surname"], - name=item["name"], - patronymic=item["patronymic"], - ) - model_ins.save() - if key == "main_disciplinelevel": - model_ins = models_name( - id=item["id"], - name=item["name"], - discipline_name_id=item["discipline_name_id"], - ) - model_ins.save() - if key == "main_staffteammember": - model_ins = models_name( - id=item["id"], - staff_position=item["staff_position"], - qualification=item["qualification"], - notes=item["notes"], - staff_member_id=item["staff_member_id"], - ) - model_ins.save() - - if key == "main_diagnosis": - nosology = None - if item.get("nosology_id"): - try: - nosology = Nosology.objects.get(pk=item["nosology_id"]) - except Nosology.DoesNotExist: - print(f"Нозологии с id {item['nosology_id']} не существует.") - model_ins = models_name( - id=item["id"], - name=item["name"], - nosology=nosology, - ) - - model_ins.save() - if key == "main_team": - model_ins = models_name( - id=item["id"], - name=item["name"], - city_id=item["city_id"], - discipline_name_id=item["discipline_name_id"], - curator_id=1, - ) - model_ins.save() - if item["staff_team_member_id"]: - staff_team_member = StaffTeamMember.objects.get( - pk=item["staff_team_member_id"] - ) - team = Team.objects.get(pk=item["id"]) - staff_team_member.team.add(team) - if key == "main_player": - diagnosis = None - if item.get("diagnosis_id"): - try: - diagnosis = Diagnosis.objects.get(pk=item["diagnosis_id"]) - except Diagnosis.DoesNotExist: - print(f"Диагноз с id {item.get('diagnosis_id')} отсутсвует.") - model_ins = models_name( - id=item["id"], - surname=item["surname"], - name=item["name"], - patronymic=item["patronymic"], - birthday=item["birthday"], - gender=item["gender"], - level_revision=item["level_revision"], - position=item["position"], - number=item["number"], - is_captain=item["is_captain"], - is_assistent=item["is_assistent"], - identity_document=item["identity_document"], - diagnosis=diagnosis, - diagnosis_id=item["diagnosis_id"], - discipline_name_id=disciplines[item["discipline_id"]][ - "discipline_name_id" - ], - discipline_level_id=disciplines[item["discipline_id"]][ - "discipline_level_id" - ], - ) - model_ins.save() - if key == "main_player_team": - player = Player.objects.get(pk=item["player_id"]) - team = Team.objects.get(pk=item["team_id"]) - player.team.add(team) - - except Exception as e: - print(f"Ошибка вставки данных {e} -> {item}") - cursor = connection.cursor() - cursor = cursor.execute(f"ALTER SEQUENCE {key}_id_seq RESTART WITH {max_id+1};") - transaction.commit() diff --git a/adaptive_hockey_federation/parser/parser.py b/adaptive_hockey_federation/parser/parser.py deleted file mode 100644 index 5da6c5e3..00000000 --- a/adaptive_hockey_federation/parser/parser.py +++ /dev/null @@ -1,108 +0,0 @@ -import json -import os -from pprint import pprint - -import click -import docx # type: ignore - -from adaptive_hockey_federation.core.config.dev_settings import ( - FIXSTURES_DIR, FIXSTURES_FILE) -from adaptive_hockey_federation.parser.docx_parser import ( - docx_parser, find_numeric_statuses) -from adaptive_hockey_federation.parser.xlsx_parser import xlsx_parser - -NUMERIC_STATUSES = "Числовые статусы следж-хоккей 02.10.203.docx" -FILES_BLACK_LIST = [ - "На мандатную комиссию", - "Именная заявка следж-хоккей Энергия Жизни Сочи", - "ФАХ Сияжар Взрослые", - "Числовые статусы следж-хоккей 02.10.203", -] -FILES_EXTENSIONS = [ - ".docx", - ".xlsx", -] -NUMERIC_STATUSES_FILE_ERROR = ( - "Не могу найти {}. Без него не" - " получиться загрузить именные заявки." - " Файл должен находиться в директории с" - " файлами для парсинга" -) - - -@click.command() -@click.option( - "-p", - "--path", - required=True, - help="Путь до папки с файлами для парсинга", -) -@click.option( - "-r", - "--result", - is_flag=True, - help="Вывод в консоль извлеченных данных и статистики", -) -def parsing_file(path: str, result: bool) -> None: - """Функция запускает парсинг файлов в рамках проекта. - Запуск через командную строку: - 'python parser.py -p(--path) путь_до_папки_с_файлами' - Вызов справки 'python parser.py -h(--help)' - """ - results_list = [] - files, numeric_statuses_file = get_all_files(path) - if numeric_statuses_file is None: - click.echo(NUMERIC_STATUSES_FILE_ERROR.format(NUMERIC_STATUSES)) - return - numeric_statuses = find_numeric_statuses(docx.Document(numeric_statuses_file)) - click.echo(f"Найдено {len(files)} файлов.") - for file in files: - if file.endswith("docx"): - results_list.extend(docx_parser(file, numeric_statuses)) - else: - results_list.extend(xlsx_parser(file)) # type: ignore - if result: - for data in results_list: - pprint(data) - - if not os.path.exists(FIXSTURES_DIR): - os.makedirs(FIXSTURES_DIR) - json.dump( - results_list, - open(FIXSTURES_FILE, "w", encoding="utf8"), - ensure_ascii=False, - indent=4, - default=str, - ) - - results_list = list(results_list) - - click.echo(f"Успешно обработано {len(files)} файлов.") - click.echo(f"Извлечено {len(results_list)} уникальных записей") - - -def get_all_files(path: str) -> tuple[list[str], str | None]: - """Функция извлекает из папки, в том числе вложенных, - список всех файлов и отдельно путь до файла с числовыми статусами. - Извлекаются только файлы с расширениями указанными в константе - FILES_EXTENSIONS (по умолчанию docx, xlsx) и не извлекает файлы, название - которых без расширения указано в списке FILES_BLACK_LIST. - """ - files = [] - numeric_statuses_filepath = None - for dirpath, dirnames, filenames in os.walk(path): - for filename in filenames: - if filename == NUMERIC_STATUSES: - numeric_statuses_filepath = os.path.join(dirpath, filename) - file, extension = os.path.splitext(filename) - if ( - not file.startswith("~") - and extension in FILES_EXTENSIONS - and file not in FILES_BLACK_LIST - ): - files.append(os.path.join(dirpath, filename)) - return files, numeric_statuses_filepath - - -if __name__ == "__main__": - parsing_file() diff --git a/adaptive_hockey_federation/parser/user_card.py b/adaptive_hockey_federation/parser/user_card.py deleted file mode 100644 index ad3d29c9..00000000 --- a/adaptive_hockey_federation/parser/user_card.py +++ /dev/null @@ -1,46 +0,0 @@ -from dataclasses import dataclass -from datetime import date -from typing import Union - - -@dataclass -class BaseUserInfo: - """Основной класс с обязательными полями.""" - - name: Union[str, None] - surname: Union[str, None] - date_of_birth: Union[date, None] - team: Union[str, None] - player_number: Union[int, None] - position: Union[str, None] - numeric_status: Union[int, None] - patronymic: Union[str, None] = None - birth_certificate: Union[str, None] = None - passport: Union[str, None] = None - classification: Union[str, None] = None - revision: Union[str, None] = None - is_assistant: bool = False - is_captain: bool = False - - def __eq__(self, other): - return all(getattr(self, attr) == getattr(other, attr) for attr in vars(self)) - - def __hash__(self): - return hash( - ( - self.name, - self.surname, - self.date_of_birth, - self.team, - self.player_number, - self.position, - self.numeric_status, - self.patronymic, - self.birth_certificate, - self.passport, - self.classification, - self.revision, - self.is_assistant, - self.is_captain, - ) - ) diff --git a/adaptive_hockey_federation/parser/xlsx_parser.py b/adaptive_hockey_federation/parser/xlsx_parser.py deleted file mode 100644 index 51046f58..00000000 --- a/adaptive_hockey_federation/parser/xlsx_parser.py +++ /dev/null @@ -1,31 +0,0 @@ -import openpyxl - -from adaptive_hockey_federation.parser.user_card import BaseUserInfo - - -def xlsx_parser(path: str) -> list[BaseUserInfo]: - """Функция парсит xlsx файлы и возвращает - игроков в виде dataclass ExcelData. - """ - players = [] - sheet_data = [] - workbook = openpyxl.load_workbook(path) - sheet = workbook.active - header = [cell.value for cell in sheet[1]] # type: ignore - for row in sheet.iter_rows(min_row=2, values_only=True): # type: ignore - sheet_data.append(dict(zip(header, row))) - for data in sheet_data: - if data.get("ФИО игрока") is not None: - player = BaseUserInfo( - team=data.get("Команда"), - name=data.get("ФИО игрока").split()[0], # type: ignore - surname=data.get("ФИО игрока").split()[1], # type: ignore - date_of_birth=data.get("Дата рождения"), - player_number=data.get("Номер игрока"), - position=data.get("Позиция"), - classification=data.get("Класс"), - revision=data.get("Пересмотр (начало сезона)"), - numeric_status=None, - ) - players.append(player.__dict__) - return players diff --git a/pyproject.toml b/pyproject.toml index 2a266ac1..ac25ba25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,6 @@ build-backend = "poetry.core.masonry.api" target-version = "py311" exclude = [ "config", - "adaptive_hockey_federation/parser/*", "*migrations/*", ".bzr", ".direnv", @@ -142,9 +141,6 @@ inline-quotes = "double" [tool.django-stubs] django_settings_module = "adaptive_hockey_federation.core.config.dev_settings" -[tool.poetry.scripts] -parser = "adaptive_hockey_federation.parser.parser:parsing_file" - [tool.pytest.ini_options] DJANGO_SETTINGS_MODULE = "adaptive_hockey_federation.core.config.test_settings" python_files = ["*_test.py"] diff --git a/requirements/develop.txt b/requirements/develop.txt index bb41838a..f83ebd67 100644 --- a/requirements/develop.txt +++ b/requirements/develop.txt @@ -1,18 +1,9 @@ -amqp==5.2.0 ; python_version >= "3.11" and python_version < "4.0" annotated-types==0.7.0 ; python_version >= "3.11" and python_version < "4.0" anyio==4.4.0 ; python_version >= "3.11" and python_version < "4.0" asgiref==3.8.1 ; python_version >= "3.11" and python_version < "4.0" -async-timeout==4.0.3 ; python_version >= "3.11" and python_full_version < "3.11.3" attrs==23.2.0 ; python_version >= "3.11" and python_version < "4.0" -billiard==4.2.0 ; python_version >= "3.11" and python_version < "4.0" -celery-singleton==0.3.1 ; python_version >= "3.11" and python_version < "4.0" -celery==5.4.0 ; python_version >= "3.11" and python_version < "4.0" -celery[redis]==5.4.0 ; python_version >= "3.11" and python_version < "4.0" certifi==2024.6.2 ; python_version >= "3.11" and python_version < "4.0" cfgv==3.4.0 ; python_version >= "3.11" and python_version < "4.0" -click-didyoumean==0.3.1 ; python_version >= "3.11" and python_version < "4.0" -click-plugins==1.1.1 ; python_version >= "3.11" and python_version < "4.0" -click-repl==0.3.0 ; python_version >= "3.11" and python_version < "4.0" click==8.1.7 ; python_version >= "3.11" and python_version < "4.0" colorama==0.4.6 ; python_version >= "3.11" and python_version < "4.0" and (platform_system == "Windows" or sys_platform == "win32") distlib==0.3.8 ; python_version >= "3.11" and python_version < "4.0" @@ -31,19 +22,16 @@ faker==26.0.0 ; python_version >= "3.11" and python_version < "4.0" fastapi-cli[standard]==0.0.5 ; python_version >= "3.11" and python_version < "4.0" fastapi[standard]==0.112.0 ; python_version >= "3.11" and python_version < "4.0" filelock==3.15.4 ; python_version >= "3.11" and python_version < "4.0" -flower==2.0.1 ; python_version >= "3.11" and python_version < "4.0" gunicorn==21.2.0 ; python_version >= "3.11" and python_version < "4.0" h11==0.14.0 ; python_version >= "3.11" and python_version < "4.0" httpcore==1.0.5 ; python_version >= "3.11" and python_version < "4.0" httptools==0.6.1 ; python_version >= "3.11" and python_version < "4.0" httpx==0.27.0 ; python_version >= "3.11" and python_version < "4.0" -humanize==4.9.0 ; python_version >= "3.11" and python_version < "4.0" identify==2.5.36 ; python_version >= "3.11" and python_version < "4.0" idna==3.7 ; python_version >= "3.11" and python_version < "4.0" inflection==0.5.1 ; python_version >= "3.11" and python_version < "4.0" iniconfig==2.0.0 ; python_version >= "3.11" and python_version < "4.0" jinja2==3.1.4 ; python_version >= "3.11" and python_version < "4.0" -kombu==5.3.7 ; python_version >= "3.11" and python_version < "4.0" markdown-it-py==3.0.0 ; python_version >= "3.11" and python_version < "4.0" markupsafe==2.1.5 ; python_version >= "3.11" and python_version < "4.0" mdurl==0.1.2 ; python_version >= "3.11" and python_version < "4.0" @@ -51,7 +39,7 @@ mypy-extensions==1.0.0 ; python_version >= "3.11" and python_version < "4.0" mypy==1.10.1 ; python_version >= "3.11" and python_version < "4.0" nodeenv==1.9.1 ; python_version >= "3.11" and python_version < "4.0" numpy==2.0.0 ; python_version >= "3.11" and python_version < "4.0" -opencv-python-headless==4.10.0.84 ; python_version >= "3.11" and python_version < "4.0" +opencv-python==4.10.0.84 ; python_version >= "3.11" and python_version < "4.0" openpyxl-stubs==0.1.25 ; python_version >= "3.11" and python_version < "4.0" openpyxl==3.1.5 ; python_version >= "3.11" and python_version < "4.0" packaging==24.1 ; python_version >= "3.11" and python_version < "4.0" @@ -60,8 +48,6 @@ pillow==10.3.0 ; python_version >= "3.11" and python_version < "4.0" platformdirs==4.2.2 ; python_version >= "3.11" and python_version < "4.0" pluggy==1.5.0 ; python_version >= "3.11" and python_version < "4.0" pre-commit==3.5.0 ; python_version >= "3.11" and python_version < "4.0" -prometheus-client==0.20.0 ; python_version >= "3.11" and python_version < "4.0" -prompt-toolkit==3.0.47 ; python_version >= "3.11" and python_version < "4.0" psycopg2-binary==2.9.9 ; python_version >= "3.11" and python_version < "4.0" pydantic-core==2.20.1 ; python_version >= "3.11" and python_version < "4.0" pydantic==2.8.2 ; python_version >= "3.11" and python_version < "4.0" @@ -74,9 +60,7 @@ python-dotenv==1.0.1 ; python_version >= "3.11" and python_version < "4.0" python-multipart==0.0.9 ; python_version >= "3.11" and python_version < "4.0" pytz==2024.1 ; python_version >= "3.11" and python_version < "4.0" pyyaml==6.0.1 ; python_version >= "3.11" and python_version < "4.0" -redis==5.0.7 ; python_version >= "3.11" and python_version < "4.0" rich==13.7.1 ; python_version >= "3.11" and python_version < "4.0" -requests==2.32.3 ; python_version >= "3.11" and python_version < "4.0" ruff==0.4.10 ; python_version >= "3.11" and python_version < "4.0" setuptools==70.1.1 ; python_version >= "3.11" and python_version < "4.0" shellingham==1.5.4 ; python_version >= "3.11" and python_version < "4.0" @@ -84,20 +68,17 @@ six==1.16.0 ; python_version >= "3.11" and python_version < "4.0" sniffio==1.3.1 ; python_version >= "3.11" and python_version < "4.0" sqlparse==0.5.0 ; python_version >= "3.11" and python_version < "4.0" starlette==0.37.2 ; python_version >= "3.11" and python_version < "4.0" -tornado==6.4.1 ; python_version >= "3.11" and python_version < "4.0" typer==0.12.3 ; python_version >= "3.11" and python_version < "4.0" types-openpyxl==3.1.4.20240626 ; python_version >= "3.11" and python_version < "4.0" types-pillow==10.2.0.20240520 ; python_version >= "3.11" and python_version < "4.0" types-python-dateutil==2.9.0.20240316 ; python_version >= "3.11" and python_version < "4.0" typing-extensions==4.12.2 ; python_version >= "3.11" and python_version < "4.0" -tzdata==2024.1 ; python_version >= "3.11" and python_version < "4.0" +tzdata==2024.1 ; python_version >= "3.11" and python_version < "4.0" and sys_platform == "win32" uritemplate==4.1.1 ; python_version >= "3.11" and python_version < "4.0" uvicorn[standard]==0.30.5 ; python_version >= "3.11" and python_version < "4.0" uvloop==0.19.0 ; (sys_platform != "win32" and sys_platform != "cygwin") and platform_python_implementation != "PyPy" and python_version >= "3.11" and python_version < "4.0" -vine==5.1.0 ; python_version >= "3.11" and python_version < "4.0" virtualenv==20.26.3 ; python_version >= "3.11" and python_version < "4.0" watchfiles==0.23.0 ; python_version >= "3.11" and python_version < "4.0" -wcwidth==0.2.13 ; python_version >= "3.11" and python_version < "4.0" websockets==12.0 ; python_version >= "3.11" and python_version < "4.0" wrapt==1.16.0 ; python_version >= "3.11" and python_version < "4.0" yadisk==3.1.0 ; python_version >= "3.11" and python_version < "4.0" diff --git a/requirements/production.txt b/requirements/production.txt index 1f8987ff..7f6f39ca 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,14 +1,5 @@ -amqp==5.2.0 ; python_version >= "3.11" and python_version < "4.0" asgiref==3.8.1 ; python_version >= "3.11" and python_version < "4.0" -async-timeout==4.0.3 ; python_version >= "3.11" and python_full_version < "3.11.3" attrs==23.2.0 ; python_version >= "3.11" and python_version < "4.0" -billiard==4.2.0 ; python_version >= "3.11" and python_version < "4.0" -celery-singleton==0.3.1 ; python_version >= "3.11" and python_version < "4.0" -celery==5.4.0 ; python_version >= "3.11" and python_version < "4.0" -celery[redis]==5.4.0 ; python_version >= "3.11" and python_version < "4.0" -click-didyoumean==0.3.1 ; python_version >= "3.11" and python_version < "4.0" -click-plugins==1.1.1 ; python_version >= "3.11" and python_version < "4.0" -click-repl==0.3.0 ; python_version >= "3.11" and python_version < "4.0" click==8.1.7 ; python_version >= "3.11" and python_version < "4.0" colorama==0.4.6 ; python_version >= "3.11" and python_version < "4.0" and (platform_system == "Windows" or sys_platform == "win32") django-environ==0.11.2 ; python_version >= "3.11" and python_version < "4" @@ -21,7 +12,6 @@ et-xmlfile==1.1.0 ; python_version >= "3.11" and python_version < "4.0" gunicorn==21.2.0 ; python_version >= "3.11" and python_version < "4.0" inflection==0.5.1 ; python_version >= "3.11" and python_version < "4.0" iniconfig==2.0.0 ; python_version >= "3.11" and python_version < "4.0" -kombu==5.3.7 ; python_version >= "3.11" and python_version < "4.0" mypy-extensions==1.0.0 ; python_version >= "3.11" and python_version < "4.0" mypy==1.10.1 ; python_version >= "3.11" and python_version < "4.0" openpyxl-stubs==0.1.25 ; python_version >= "3.11" and python_version < "4.0" @@ -30,23 +20,17 @@ packaging==24.1 ; python_version >= "3.11" and python_version < "4.0" phonenumbers==8.13.39 ; python_version >= "3.11" and python_version < "4.0" pillow==10.3.0 ; python_version >= "3.11" and python_version < "4.0" pluggy==1.5.0 ; python_version >= "3.11" and python_version < "4.0" -prompt-toolkit==3.0.47 ; python_version >= "3.11" and python_version < "4.0" psycopg2-binary==2.9.9 ; python_version >= "3.11" and python_version < "4.0" pytest-django==4.8.0 ; python_version >= "3.11" and python_version < "4.0" pytest-subtests==0.12.1 ; python_version >= "3.11" and python_version < "4.0" pytest==8.2.2 ; python_version >= "3.11" and python_version < "4.0" -python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "4.0" pytz==2024.1 ; python_version >= "3.11" and python_version < "4.0" pyyaml==6.0.1 ; python_version >= "3.11" and python_version < "4.0" -redis==5.0.7 ; python_version >= "3.11" and python_version < "4.0" -six==1.16.0 ; python_version >= "3.11" and python_version < "4.0" sqlparse==0.5.0 ; python_version >= "3.11" and python_version < "4.0" types-openpyxl==3.1.4.20240626 ; python_version >= "3.11" and python_version < "4.0" types-python-dateutil==2.9.0.20240316 ; python_version >= "3.11" and python_version < "4.0" typing-extensions==4.12.2 ; python_version >= "3.11" and python_version < "4.0" -tzdata==2024.1 ; python_version >= "3.11" and python_version < "4.0" +tzdata==2024.1 ; python_version >= "3.11" and python_version < "4.0" and sys_platform == "win32" uritemplate==4.1.1 ; python_version >= "3.11" and python_version < "4.0" -vine==5.1.0 ; python_version >= "3.11" and python_version < "4.0" -wcwidth==0.2.13 ; python_version >= "3.11" and python_version < "4.0" wrapt==1.16.0 ; python_version >= "3.11" and python_version < "4.0" yadisk==3.1.0 ; python_version >= "3.11" and python_version < "4.0"