From 164b30beed4893f53c88102be95961001329c9aa Mon Sep 17 00:00:00 2001 From: Jai Date: Mon, 4 Mar 2024 15:20:50 +0530 Subject: [PATCH 1/5] UNSTABLE -> Moving Datahub to Participant / Moving Dataset to its own app --- accounts/views.py | 7 +- connectors/models.py | 2 +- connectors/serializers.py | 4 +- connectors/tests/test_connectors_list.py | 2 +- connectors/views.py | 2 +- core/settings.py | 3 +- core/views.py | 2 +- datahub/models.py | 24 +- datahub/serializers.py | 4 +- datasets/__init__.py | 0 datasets/admin.py | 3 + datasets/apps.py | 6 + datasets/migrations/__init__.py | 0 datasets/models.py | 66 + datasets/serializers.py | 168 + datasets/tests.py | 3 + datasets/urls.py | 16 + datasets/views.py | 421 ++ microsite/serializers.py | 6 +- microsite/tests/test_connectors_views.py | 2 +- microsite/tests/test_dataset_json_response.py | 2 +- microsite/tests/test_get_json_from_api_key.py | 2 +- microsite/tests/test_policy_view.py | 2 +- microsite/views.py | 6 +- participant/models.py | 515 ++- participant/serializers.py | 1491 ++++++- .../tests/test_create_new_req_views.py | 2 +- .../tests/test_support_ticket_views.py | 2 +- participant/tests/test_views.py | 6 +- participant/tests/test_views_support.py | 2 +- participant/urls.py | 59 + participant/views.py | 3686 ++++++++++++++++- utils/authentication_services.py | 4 +- utils/embeddings_creation.py | 2 +- 34 files changed, 6390 insertions(+), 132 deletions(-) create mode 100644 datasets/__init__.py create mode 100644 datasets/admin.py create mode 100644 datasets/apps.py create mode 100644 datasets/migrations/__init__.py create mode 100644 datasets/models.py create mode 100644 datasets/serializers.py create mode 100644 datasets/tests.py create mode 100644 datasets/urls.py create mode 100644 datasets/views.py diff --git a/accounts/views.py b/accounts/views.py index c8922f1c..95828d73 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -44,15 +44,16 @@ date_formater, read_contents_from_csv_or_xlsx_file, ) -from datahub.models import ( +from participant.models import ( DatahubDocuments, - Datasets, DatasetV2, DatasetV2File, Organization, UserOrganizationMap, ) -from datahub.serializers import ( + +from datasets.models import Datasets +from participant.serializers import ( DatahubDatasetsSerializer, DatahubDatasetsV2Serializer, DatahubThemeSerializer, diff --git a/connectors/models.py b/connectors/models.py index 6ce0af80..7eeade27 100644 --- a/connectors/models.py +++ b/connectors/models.py @@ -5,7 +5,7 @@ from accounts.models import User from core import settings from core.base_models import TimeStampMixin -from datahub.models import DatasetV2File +from participant.models import DatasetV2File # Create your models here. diff --git a/connectors/serializers.py b/connectors/serializers.py index 872bd787..fa859bd0 100644 --- a/connectors/serializers.py +++ b/connectors/serializers.py @@ -9,8 +9,8 @@ from connectors.models import Connectors, ConnectorsMap from core import settings from core.constants import Constants -from datahub.models import DatasetV2, DatasetV2File, Organization, UserOrganizationMap -from datahub.serializers import DatasetV2FileSerializer +from participant.models import DatasetV2, DatasetV2File, Organization, UserOrganizationMap +from participant.serializers import DatasetV2FileSerializer from django.db.models import Subquery, Min, Count from django.db import models diff --git a/connectors/tests/test_connectors_list.py b/connectors/tests/test_connectors_list.py index 04c76436..73d89a20 100644 --- a/connectors/tests/test_connectors_list.py +++ b/connectors/tests/test_connectors_list.py @@ -2,7 +2,7 @@ from django.test import Client, TestCase from rest_framework import status import json -from datahub.models import DatasetV2, Organization, UserOrganizationMap, DatasetV2File +from participant.models import DatasetV2, Organization, UserOrganizationMap, DatasetV2File from accounts.models import User, UserRole from connectors.models import Connectors, ConnectorsMap from participant.tests.test_util import TestUtils diff --git a/connectors/views.py b/connectors/views.py index 4061b509..7c85d512 100644 --- a/connectors/views.py +++ b/connectors/views.py @@ -25,7 +25,7 @@ from core import settings from core.constants import Constants from core.utils import CustomPagination -from datahub.models import Datasets +from datasets.models import Datasets from utils.authentication_services import authenticate_user from utils.jwt_services import http_request_mutation from rest_framework.exceptions import ValidationError diff --git a/core/settings.py b/core/settings.py index 6216ab11..070112de 100644 --- a/core/settings.py +++ b/core/settings.py @@ -65,6 +65,7 @@ # custom apps "accounts", "datahub", + "datasets", "participant", "microsite", "connectors", @@ -129,7 +130,7 @@ } # Password validation -# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators +# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-v\alidators AUTH_PASSWORD_VALIDATORS = [ { diff --git a/core/views.py b/core/views.py index a5a0b3d2..7416d1d8 100644 --- a/core/views.py +++ b/core/views.py @@ -26,7 +26,7 @@ from rest_framework_simplejwt.tokens import AccessToken, RefreshToken from core.constants import Constants -from datahub.models import DatasetV2File, UsagePolicy, UserOrganizationMap +from participant.models import DatasetV2File, UsagePolicy, UserOrganizationMap from utils.jwt_services import http_request_mutation # @http_request_mutation diff --git a/datahub/models.py b/datahub/models.py index 9eeb23a1..ee8d7f6c 100644 --- a/datahub/models.py +++ b/datahub/models.py @@ -77,7 +77,7 @@ class UserOrganizationMap(TimeStampMixin): """UserOrganizationMap model for mapping User and Organization model""" id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, on_delete=models.CASCADE) + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="datahub_org_map_user") organization = models.ForeignKey(Organization, on_delete=models.CASCADE) @@ -203,17 +203,17 @@ def __init__(self, dataset_name, source): # def size(self, name): # path = self.path(name) # return os.path.getsize(path) - + def exists(self, name): """ Check if a file with the given name already exists in the storage. """ return os.path.exists(name) - + def url(self, url): return url - def _save(self, name, content): + def _save(self, name, content): # Save file to a directory outside MEDIA_ROOT full_path = os.path.join(settings.DATASET_FILES_URL, name) directory = os.path.dirname(full_path) @@ -260,7 +260,7 @@ class DatasetV2File(TimeStampMixin): def dataset_directory_path(instance, filename): # file will be uploaded to MEDIA_ROOT/user_/ return f"{settings.DATASET_FILES_URL}/{instance.dataset.name}/{instance.source}/{filename}" - + def get_upload_path(instance, filename): return f"{instance.dataset.name}/{instance.source}/{filename}" @@ -268,12 +268,12 @@ def save(self, *args, **kwargs): # set the user_id before saving storage = CustomStorage(self.dataset.name, self.source) self.file.storage = storage # type: ignore - + # if self.file: # # Get the file size # size = self.file.size # self.file_size = size - + super().save(*args, **kwargs) SOURCES = [ @@ -291,7 +291,7 @@ def save(self, *args, **kwargs): standardised_configuration = models.JSONField(default = dict) accessibility = models.CharField(max_length=255, null=True, choices=USAGE_POLICY_APPROVAL_STATUS, default="public") connection_details = models.JSONField(default=dict, null=True) - + class UsagePolicy(TimeStampMixin): """ Policy documentation Model. @@ -324,7 +324,7 @@ def __str__(self) -> str: class ResourceFile(TimeStampMixin): """ - Resource Files Model -- Has a one to many relation + Resource Files Model -- Has a one to many relation -- 1 resource can have multiple resource files. """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) @@ -338,7 +338,7 @@ class ResourceFile(TimeStampMixin): embeddings_status_reason = models.CharField(max_length=1000, null=True) def __str__(self) -> str: return self.file.name - + class DatasetV2FileReload(TimeStampMixin): dataset_file = models.ForeignKey(DatasetV2File, on_delete=models.CASCADE, related_name="dataset_file") @@ -385,7 +385,7 @@ class LangchainPgCollection(models.Model): # resource_file = models.ForeignKey(ResourceFile, on_delete=models.PROTECT, related_name="resource_file_collections") class Meta: - db_table = 'langchain_pg_collection' + db_table = 'langchain_pg_collection_datahub' class LangchainPgEmbedding(models.Model): @@ -397,7 +397,7 @@ class LangchainPgEmbedding(models.Model): uuid = models.UUIDField(primary_key=True) class Meta: - db_table = 'langchain_pg_embedding' + db_table = 'langchain_pg_embedding_datahub' # def __str__(self): # return f"LangchainPgEmbedding(uuid={self.uuid}, document={self.document})" diff --git a/datahub/serializers.py b/datahub/serializers.py index 2ac68e55..26e87e2d 100644 --- a/datahub/serializers.py +++ b/datahub/serializers.py @@ -38,7 +38,9 @@ StandardisationTemplate, UserOrganizationMap, ) -from participant.models import Connectors, SupportTicket +# from participant.models import Connectors, SupportTicket +# TODO - REMOVED IMPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT +from participant.models import SupportTicket from utils.custom_exceptions import NotFoundException from utils.embeddings_creation import VectorDBBuilder from utils.file_operations import create_directory, move_directory diff --git a/datasets/__init__.py b/datasets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/datasets/admin.py b/datasets/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/datasets/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/datasets/apps.py b/datasets/apps.py new file mode 100644 index 00000000..e924fb4e --- /dev/null +++ b/datasets/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class DatasetsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'datasets' diff --git a/datasets/migrations/__init__.py b/datasets/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/datasets/models.py b/datasets/models.py new file mode 100644 index 00000000..f60147c5 --- /dev/null +++ b/datasets/models.py @@ -0,0 +1,66 @@ +from django.db import models +import os +import uuid +from datetime import timedelta +from email.mime import application + +from django.conf import settings +from django.core.files.storage import Storage +from django.db import models +from django.utils import timezone +from pgvector.django import VectorField + +from accounts.models import User +from core.base_models import TimeStampMixin +from core.constants import Constants +from participant.models import UserOrganizationMap +from utils.validators import ( + validate_25MB_file_size, + validate_file_size, + validate_image_type, +) + + +APPROVAL_STATUS = ( + ("approved", "approved"), + ("rejected", "rejected"), + ("for_review", "for_review"), +) + +def auto_str(cls): + def __str__(self): + return "%s" % (", ".join("%s=%s" % item for item in vars(self).items())) + + cls.__str__ = __str__ + return cls + +# Create your models here. +@auto_str +class Datasets(TimeStampMixin): + """Datasets model of all the users""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.PROTECT) + name = models.CharField(max_length=255, unique=True) + description = models.CharField(max_length=500) + category = models.JSONField() + geography = models.CharField(max_length=255, blank=True) + crop_detail = models.CharField(max_length=255, null=True, blank=True) # field should update + constantly_update = models.BooleanField(default=False) + age_of_date = models.CharField(max_length=255, null=True, blank=True) + data_capture_start = models.DateTimeField(null=True, blank=True) + data_capture_end = models.DateTimeField(null=True, blank=True) + dataset_size = models.CharField(max_length=255, null=True, blank=True) + connector_availability = models.CharField(max_length=255, null=True, blank=True) + sample_dataset = models.FileField( + upload_to=settings.SAMPLE_DATASETS_URL, + blank=True, + ) + status = models.BooleanField(default=True) + approval_status = models.CharField(max_length=255, null=True, choices=APPROVAL_STATUS, default="for_review") + is_enabled = models.BooleanField(default=True) + is_public = models.BooleanField(default=True) + remarks = models.CharField(max_length=1000, null=True, blank=True) + + class Meta: + indexes = [models.Index(fields=["name"])] \ No newline at end of file diff --git a/datasets/serializers.py b/datasets/serializers.py new file mode 100644 index 00000000..95f1e892 --- /dev/null +++ b/datasets/serializers.py @@ -0,0 +1,168 @@ +import json +import logging +import os +import re +import secrets +import shutil +import string +import uuid +from urllib.parse import quote +from django.core.files.base import ContentFile + +import plazy +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.validators import URLValidator +from django.db.models import Count, Prefetch, Q +from django.utils.translation import gettext as _ +from requests import Response +from rest_framework import serializers, status + +from accounts import models +from accounts.models import User, UserRole +from accounts.serializers import ( + UserCreateSerializer, + UserRoleSerializer, + UserSerializer, +) +from core.constants import Constants +from participant.models import ( + DatahubDocuments, + DatasetV2, + DatasetV2File, + Organization, + Resource, + ResourceFile, + ResourceUsagePolicy, + StandardisationTemplate, + UserOrganizationMap, +) + +from datasets.models import Datasets + + +# TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT +from participant.models import SupportTicket +from utils.custom_exceptions import NotFoundException +from utils.embeddings_creation import VectorDBBuilder +from utils.file_operations import create_directory, move_directory +from utils.string_functions import check_special_chars +from utils.validators import ( + validate_dataset_size, + validate_dataset_type, + validate_document_type, + validate_file_size, + validate_image_type, +) +from utils.youtube_helper import get_youtube_url + +from participant.models import ( # Conversation, + Category, + DatasetSubCategoryMap, + LangchainPgCollection, + LangchainPgEmbedding, + Messages, + Policy, + ResourceSubCategoryMap, + SubCategory, + UsagePolicy, +) + + +class DatasetSerializer(serializers.ModelSerializer): + """_summary_ + + Args: + serializers (_type_): _description_ + """ + + def validate_sample_dataset(self, value): + """ + Validator function to check the file size limit. + """ + MAX_FILE_SIZE = ( + Constants.MAX_PUBLIC_FILE_SIZE if self.initial_data.get("is_public") else Constants.MAX_FILE_SIZE + ) + filesize = value.size + if filesize > MAX_FILE_SIZE: + raise ValidationError( + _("You cannot upload a file more than %(value)s MB"), + params={"value": MAX_FILE_SIZE / 1048576}, + ) + return value + + class Meta: + """_summary_""" + + model = Datasets + fields = [ + "user_map", + "name", + "description", + "category", + "geography", + "crop_detail", + "constantly_update", + "dataset_size", + "connector_availability", + "age_of_date", + "sample_dataset", + "data_capture_start", + "data_capture_end", + "remarks", + "is_enabled", + "approval_status", + "is_public", + ] + + +class DatahubDatasetsSerializer(serializers.ModelSerializer): + class OrganizationDatsetsListRetriveSerializer(serializers.ModelSerializer): + class Meta: + model = Organization + fields = [ + "org_email", + "org_description", + "name", + "logo", + "address", + "phone_number", + ] + + class UserDatasetSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ["last_name", "first_name", "email", "on_boarded_by"] + + user_id = serializers.PrimaryKeyRelatedField( + queryset=models.User.objects.all(), required=True, source="user_map.user" + ) + organization_id = serializers.PrimaryKeyRelatedField( + queryset=Organization.objects.all(), + allow_null=True, + required=False, + source="user_map.organization", + ) + + organization = OrganizationDatsetsListRetriveSerializer( + required=False, allow_null=True, read_only=True, source="user_map.organization" + ) + user = UserDatasetSerializer(required=False, allow_null=True, read_only=True, source="user_map.user") + + class Meta: + model = Datasets + fields = Constants.ALL + + +class DatasetUpdateSerializer(serializers.ModelSerializer): + """_summary_ + + Args: + serializers (_type_): _description_ + """ + + class Meta: + """_summary_""" + + model = Datasets + fields = Constants.ALL \ No newline at end of file diff --git a/datasets/tests.py b/datasets/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/datasets/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/datasets/urls.py b/datasets/urls.py new file mode 100644 index 00000000..65bc0f84 --- /dev/null +++ b/datasets/urls.py @@ -0,0 +1,16 @@ +from posixpath import basename +from sys import settrace + +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from core import settings +from core.constants import Constants +from datasets.views import DatasetsViewSetV2 + +router = DefaultRouter() +router.register(r"datasets", DatasetsViewSetV2, basename=Constants.PARTICIPANT) + +urlpatterns = [ + path("v2/", include(router.urls)), +] \ No newline at end of file diff --git a/datasets/views.py b/datasets/views.py new file mode 100644 index 00000000..d838836e --- /dev/null +++ b/datasets/views.py @@ -0,0 +1,421 @@ +import json +import logging +import operator +import os +from functools import reduce +from django.db.models import Q +from django.shortcuts import render +from jsonschema import ValidationError +from rest_framework import generics, pagination, status, viewsets +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError +from rest_framework.parsers import JSONParser, MultiPartParser +from rest_framework.response import Response +from rest_framework.viewsets import GenericViewSet, ViewSet + +from accounts.models import User, UserRole +from core.constants import Constants, NumericalConstants +from core.utils import ( + CustomPagination, + Utils, + csv_and_xlsx_file_validatation, + read_contents_from_csv_or_xlsx_file, +) +from datasets.models import ( + Datasets, + UserOrganizationMap, +) +from datasets.serializers import ( + DatasetSerializer, + DatasetUpdateSerializer, + DatahubDatasetsSerializer, +) + +from utils import custom_exceptions, file_operations, string_functions, validators +from utils.jwt_services import http_request_mutation +LOGGER = logging.getLogger(__name__) +con = None + + +# Create your views here. +class DatasetsViewSetV2(GenericViewSet): + """ + This class handles the participant Datsets CRUD operations. + """ + + parser_class = JSONParser + serializer_class = DatasetSerializer + queryset = Datasets + pagination_class = CustomPagination + + def perform_create(self, serializer): + """ + This function performs the create operation of requested serializer. + Args: + serializer (_type_): serializer class object. + + Returns: + _type_: Returns the saved details. + """ + return serializer.save() + + def trigger_email(self, request, template, to_email, subject, first_name, last_name, dataset_name): + # trigger email to the participant as they are being added + try: + datahub_admin = User.objects.filter(role_id=1).first() + admin_full_name = string_functions.get_full_name(datahub_admin.first_name, datahub_admin.last_name) + participant_full_name = string_functions.get_full_name(first_name, last_name) + + data = { + "datahub_name": os.environ.get("DATAHUB_NAME", "datahub_name"), + "participant_admin_name": participant_full_name, + "datahub_admin": admin_full_name, + "dataset_name": dataset_name, + "datahub_site": os.environ.get("DATAHUB_SITE", "datahub_site"), + } + + email_render = render(request, template, data) + mail_body = email_render.content.decode("utf-8") + Utils().send_email( + to_email=to_email, + content=mail_body, + subject=subject + os.environ.get("DATAHUB_NAME", "datahub_name"), + ) + + except Exception as error: + LOGGER.error(error, exc_info=True) + + def create(self, request, *args, **kwargs): + """POST method: create action to save an object by sending a POST request""" + setattr(request.data, "_mutable", True) + data = request.data + + if not data.get("is_public"): + if not csv_and_xlsx_file_validatation(request.data.get(Constants.SAMPLE_DATASET)): + return Response( + { + Constants.SAMPLE_DATASET: [ + "Invalid Sample dataset file (or) Atleast 5 rows should be available. please upload valid file" + ] + }, + 400, + ) + try: + data[Constants.APPROVAL_STATUS] = Constants.APPROVED + serializer = self.get_serializer(data=data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @http_request_mutation + def list(self, request, *args, **kwargs): + """GET method: query all the list of objects from the Product model""" + try: + data = [] + user_id = request.META.get(Constants.USER_ID) + others = request.data.get(Constants.OTHERS) + filters = {Constants.USER_MAP_USER: user_id} if user_id and not others else {} + exclude = {Constants.USER_MAP_USER: user_id} if others else {} + if exclude or filters: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, status=True, **filters) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + page = self.paginate_queryset(data) + participant_serializer = DatahubDatasetsSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + def retrieve(self, request, pk): + """GET method: retrieve an object or instance of the Product model""" + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, status=True, id=pk) + .all() + ) + participant_serializer = DatahubDatasetsSerializer(data, many=True) + if participant_serializer.data: + data = participant_serializer.data[0] + if not data.get("is_public"): + data[Constants.CONTENT] = read_contents_from_csv_or_xlsx_file(data.get(Constants.SAMPLE_DATASET)) + return Response(data, status=status.HTTP_200_OK) + return Response({}, status=status.HTTP_200_OK) + + def update(self, request, *args, **kwargs): + """PUT method: update or send a PUT request on an object of the Product model""" + setattr(request.data, "_mutable", True) + data = request.data + data = {key: value for key, value in data.items() if value != "null"} + if not data.get("is_public"): + if data.get(Constants.SAMPLE_DATASET): + if not csv_and_xlsx_file_validatation(data.get(Constants.SAMPLE_DATASET)): + return Response( + { + Constants.SAMPLE_DATASET: [ + "Invalid Sample dataset file (or) Atleast 5 rows should be available. please upload valid file" + ] + }, + 400, + ) + category = data.get(Constants.CATEGORY) + if category: + data[Constants.CATEGORY] = json.loads(category) if isinstance(category, str) else category + instance = self.get_object() + + # trigger email to the participant + user_map_queryset = UserOrganizationMap.objects.select_related(Constants.USER).get(id=instance.user_map_id) + user_obj = user_map_queryset.user + + # reset the approval status b/c the user modified the dataset after an approval + if getattr(instance, Constants.APPROVAL_STATUS) == Constants.APPROVED and ( + user_obj.role_id == 3 or user_obj.role_id == 4 + ): + data[Constants.APPROVAL_STATUS] = Constants.AWAITING_REVIEW + + serializer = DatasetUpdateSerializer(instance, data=data, partial=True) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + + data = request.data + + if data.get(Constants.APPROVAL_STATUS) == Constants.APPROVED: + self.trigger_email( + request, + "datahub_admin_approves_dataset.html", + user_obj.email, + Constants.APPROVED_NEW_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + elif data.get(Constants.APPROVAL_STATUS) == Constants.REJECTED: + self.trigger_email( + request, + "datahub_admin_rejects_dataset.html", + user_obj.email, + Constants.REJECTED_NEW_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + elif data.get(Constants.IS_ENABLED) == str(True) or data.get(Constants.IS_ENABLED) == str("true"): + self.trigger_email( + request, + "datahub_admin_enables_dataset.html", + user_obj.email, + Constants.ENABLE_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + elif data.get(Constants.IS_ENABLED) == str(False) or data.get(Constants.IS_ENABLED) == str("false"): + self.trigger_email( + request, + "datahub_admin_disables_dataset.html", + user_obj.email, + Constants.DISABLE_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + return Response(serializer.data, status=status.HTTP_201_CREATED) + + def destroy(self, request, pk): + """DELETE method: delete an object""" + dataset = self.get_object() + dataset.status = False + self.perform_create(dataset) + return Response(status=status.HTTP_204_NO_CONTENT) + + @action(detail=False, methods=["post"]) + def dataset_filters(self, request, *args, **kwargs): + """This function get the filter args in body. based on the filter args orm filters the data.""" + data = request.data + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + user_id = data.pop(Constants.USER_ID, "") + categories = data.pop(Constants.CATEGORY, None) + exclude, filters = {}, {} + if others: + exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + else: + filters = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + + try: + if categories is not None: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(status=True, **data, **filters) + .filter( + reduce( + operator.or_, + (Q(category__contains=cat) for cat in categories), + ) + ) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + + else: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(status=True, **data, **filters) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + return Response(f"Invalid filter fields: {list(request.data.keys())}", status=500) + + page = self.paginate_queryset(data) + participant_serializer = DatahubDatasetsSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + + @action(detail=False, methods=["post"]) + @http_request_mutation + def filters_data(self, request, *args, **kwargs): + """This function provides the filters data""" + try: + data = request.data + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + user_id = data.pop(Constants.USER_ID, "") + + #### + + org_id = request.META.pop(Constants.ORG_ID, "") + others = request.META.pop(Constants.OTHERS, "") + user_id = request.META.pop(Constants.USER_ID, "") + + exclude, filters = {}, {} + if others: + exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + filters = {Constants.APPROVAL_STATUS: Constants.APPROVED} + else: + filters = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + try: + geography = ( + Datasets.objects.values_list(Constants.GEOGRAPHY, flat=True) + .filter(status=True, **filters) + .exclude(geography="null") + .exclude(geography__isnull=True) + .exclude(geography="") + .exclude(**exclude) + .all() + .distinct() + ) + crop_detail = ( + Datasets.objects.values_list(Constants.CROP_DETAIL, flat=True) + .filter(status=True, **filters) + .exclude(crop_detail="null") + .exclude(crop_detail__isnull=True) + .exclude(crop_detail="") + .exclude(**exclude) + .all() + .distinct() + ) + if os.path.exists(Constants.CATEGORIES_FILE): + with open(Constants.CATEGORIES_FILE, "r") as json_obj: + category_detail = json.loads(json_obj.read()) + else: + category_detail = [] + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + return Response(f"Invalid filter fields: {list(request.data.keys())}", status=500) + return Response( + { + "geography": geography, + "crop_detail": crop_detail, + "category_detail": category_detail, + }, + status=200, + ) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["post"]) + @http_request_mutation + def search_datasets(self, request, *args, **kwargs): + data = request.data + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + user_id = data.pop(Constants.USER_ID, "") + + org_id = request.META.pop(Constants.ORG_ID, "") + others = request.META.pop(Constants.OTHERS, "") + user_id = request.META.pop(Constants.USER_ID, "") + + search_pattern = data.pop(Constants.SEARCH_PATTERNS, "") + exclude, filters = {}, {} + + if others: + exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + filters = {Constants.NAME_ICONTAINS: search_pattern} if search_pattern else {} + else: + filters = ( + { + Constants.USER_MAP_ORGANIZATION: org_id, + Constants.NAME_ICONTAINS: search_pattern, + } + if org_id + else {} + ) + try: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, status=True, **data, **filters) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + return Response( + f"Invalid filter fields: {list(request.data.keys())}", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + page = self.paginate_queryset(data) + participant_serializer = DatahubDatasetsSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) \ No newline at end of file diff --git a/microsite/serializers.py b/microsite/serializers.py index ed0e864b..46d11624 100644 --- a/microsite/serializers.py +++ b/microsite/serializers.py @@ -11,10 +11,10 @@ from connectors.serializers import OrganizationRetriveSerializer from core import settings from core.utils import Constants -from datahub.models import ( +from datasets.models import Datasets +from participant.models import ( Category, DatahubDocuments, - Datasets, DatasetV2, LangchainPgCollection, LangchainPgEmbedding, @@ -26,7 +26,7 @@ SubCategory, UserOrganizationMap, ) -from datahub.serializers import CategorySerializer, DatasetV2FileSerializer +from participant.serializers import CategorySerializer, DatasetV2FileSerializer from .models import FeedBack diff --git a/microsite/tests/test_connectors_views.py b/microsite/tests/test_connectors_views.py index 198666ae..28365fdd 100644 --- a/microsite/tests/test_connectors_views.py +++ b/microsite/tests/test_connectors_views.py @@ -9,7 +9,7 @@ from accounts.models import User, UserRole from connectors.models import Connectors, ConnectorsMap -from datahub.models import DatasetV2, DatasetV2File, Organization, UserOrganizationMap +from participant.models import DatasetV2, DatasetV2File, Organization, UserOrganizationMap from participant.tests.test_util import TestUtils LOGGER = logging.getLogger(__name__) diff --git a/microsite/tests/test_dataset_json_response.py b/microsite/tests/test_dataset_json_response.py index a6f5e92a..413d81cb 100644 --- a/microsite/tests/test_dataset_json_response.py +++ b/microsite/tests/test_dataset_json_response.py @@ -1,7 +1,7 @@ from rest_framework.reverse import reverse from django.test import Client, TestCase import json -from datahub.models import DatasetV2, Organization, UserOrganizationMap, DatasetV2File +from participant.models import DatasetV2, Organization, UserOrganizationMap, DatasetV2File from accounts.models import User, UserRole from participant.tests.test_util import TestUtils from _pytest.monkeypatch import MonkeyPatch diff --git a/microsite/tests/test_get_json_from_api_key.py b/microsite/tests/test_get_json_from_api_key.py index 9bde54ae..6907e750 100644 --- a/microsite/tests/test_get_json_from_api_key.py +++ b/microsite/tests/test_get_json_from_api_key.py @@ -1,7 +1,7 @@ from rest_framework.reverse import reverse from django.test import Client, TestCase import json -from datahub.models import DatasetV2, Organization, UserOrganizationMap, DatasetV2File,UsagePolicy +from participant.models import DatasetV2, Organization, UserOrganizationMap, DatasetV2File,UsagePolicy from accounts.models import User, UserRole from participant.tests.test_util import TestUtils from _pytest.monkeypatch import MonkeyPatch diff --git a/microsite/tests/test_policy_view.py b/microsite/tests/test_policy_view.py index c7134e63..9bf82c0d 100644 --- a/microsite/tests/test_policy_view.py +++ b/microsite/tests/test_policy_view.py @@ -1,4 +1,4 @@ -from datahub.models import Policy +from participant.models import Policy from rest_framework.reverse import reverse from django.test import Client, TestCase import json diff --git a/microsite/views.py b/microsite/views.py index ff1f6cca..7f1541db 100644 --- a/microsite/views.py +++ b/microsite/views.py @@ -38,10 +38,10 @@ generate_hash_key_for_dashboard, read_contents_from_csv_or_xlsx_file, ) -from datahub.models import ( +from datasets.models import Datasets +from participant.models import ( Category, DatahubDocuments, - Datasets, DatasetV2, DatasetV2File, Messages, @@ -53,7 +53,7 @@ UsagePolicy, UserOrganizationMap, ) -from datahub.serializers import ( +from participant.serializers import ( CategorySerializer, DatahubDatasetsV2Serializer, DatasetV2Serializer, diff --git a/participant/models.py b/participant/models.py index 6620dafd..377a7b6d 100644 --- a/participant/models.py +++ b/participant/models.py @@ -1,15 +1,15 @@ +import os import uuid -from sre_constants import CATEGORY -from unicodedata import category -# import black +from django.core.files.storage import Storage from django.db import models +from pgvector.django import VectorField from accounts.models import User from core import settings from core.base_models import TimeStampMixin -from datahub.models import Datasets, Organization, UserOrganizationMap -from utils.validators import validate_file_size +from core.constants import Constants +from utils.validators import validate_file_size, validate_image_type, validate_25MB_file_size CATEGORY = ( ("connectors", "connectors"), @@ -20,6 +20,45 @@ ("certificate", "certificate"), ) +CONNECTOR_TYPE = (("MYSQL", "MYSQL"), ("MONGODB", "MONDODB"), ("CSV", "CSV")) +APPROVAL_STATUS = ( + ("approved", "approved"), + ("rejected", "rejected"), + ("for_review", "for_review"), +) +USAGE_POLICY_REQUEST_STATUS = ( + ("approved", "approved"), + ("rejected", "rejected"), + ("requested", "requested") + +) +USAGE_POLICY_API_TYPE = ( + ("dataset_file", "dataset_file"), + ("api", "api") +) + +RESOURCE_USAGE_POLICY_API_TYPE = ( + ("resource", "resource"), + ("embeddings", "embeddings") +) +RESOURCE_URL_TYPE = ( + ("youtube", "youtube"), + ("pdf", "pdf"), + ("file", "file"), + ("website", "website"), + ("api", "api") +) + +USAGE_POLICY_APPROVAL_STATUS = ( + ("public", "public"), + ("registered", "registered"), + ("private", "private"), +) +EMBEDDINGS_STATUS = ( + ("completed", "completed"), + ("in-progress", "in-progress"), + ("failed", "failed"), +) STATUS = (("open", "open"), ("hold", "hold"), ("closed", "closed")) @@ -34,6 +73,52 @@ def __str__(self): return cls + + +@auto_str +class Organization(TimeStampMixin): + """Organization model + + status: + active = 1 + inactive = 0 + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=255) + org_email = models.EmailField(max_length=255, unique=True) + address = models.JSONField() + phone_number = models.CharField(max_length=50, null=True, blank=True) + logo = models.ImageField( + upload_to=settings.ORGANIZATION_IMAGES_URL, + null=True, + blank=True, + validators=[validate_file_size, validate_image_type], + ) + hero_image = models.ImageField( + upload_to=settings.ORGANIZATION_IMAGES_URL, + null=True, + blank=True, + validators=[validate_file_size, validate_image_type], + ) + org_description = models.TextField(max_length=512, null=True, blank=True) + website = models.CharField(max_length=255, null=True, blank=True) + status = models.BooleanField(default=True) + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + +@auto_str +class UserOrganizationMap(TimeStampMixin): + """UserOrganizationMap model for mapping User and Organization model""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey(User, on_delete=models.CASCADE) + organization = models.ForeignKey(Organization, on_delete=models.CASCADE) + + + @auto_str class SupportTicket(TimeStampMixin): """SupportTicket model of all the participant users""" @@ -85,42 +170,43 @@ class Project(TimeStampMixin): status = models.BooleanField(default=True) -@auto_str -class Connectors(TimeStampMixin): - """Connectors model of all the users""" +# TODO - UNCOMMENT MODEL CURRENTLY COMMENTED TO AVOID CIRCULAR IMPORT +# @auto_str +# class Connectors(TimeStampMixin): +# """Connectors model of all the users""" +# +# id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) +# user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.PROTECT, blank=True, default="") +# project = models.ForeignKey(Project, on_delete=models.PROTECT, null=True) +# department = models.ForeignKey( +# Department, on_delete=models.PROTECT, default="e459f452-2b4b-4129-ba8b-1e1180c87888" +# ) +# dataset = models.ForeignKey(Datasets, on_delete=models.PROTECT) +# connector_name = models.CharField(max_length=255, unique=True) +# connector_type = models.CharField(max_length=255) +# connector_description = models.CharField(max_length=500, null=True, blank=True) +# docker_image_url = models.CharField(max_length=255) +# application_port = models.IntegerField() +# certificate = models.FileField( +# upload_to=settings.CONNECTORS_CERTIFICATE_URL, +# blank=True, +# validators=[validate_file_size], +# ) +# usage_policy = models.CharField(max_length=255) +# status = models.BooleanField(default=True) +# connector_status = models.CharField(max_length=255, default="install certificate") - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.PROTECT, blank=True, default="") - project = models.ForeignKey(Project, on_delete=models.PROTECT, null=True) - department = models.ForeignKey( - Department, on_delete=models.PROTECT, default="e459f452-2b4b-4129-ba8b-1e1180c87888" - ) - dataset = models.ForeignKey(Datasets, on_delete=models.PROTECT) - connector_name = models.CharField(max_length=255, unique=True) - connector_type = models.CharField(max_length=255) - connector_description = models.CharField(max_length=500, null=True, blank=True) - docker_image_url = models.CharField(max_length=255) - application_port = models.IntegerField() - certificate = models.FileField( - upload_to=settings.CONNECTORS_CERTIFICATE_URL, - blank=True, - validators=[validate_file_size], - ) - usage_policy = models.CharField(max_length=255) - status = models.BooleanField(default=True) - connector_status = models.CharField(max_length=255, default="install certificate") - - -@auto_str -class ConnectorsMap(TimeStampMixin): - """ConnectorsMap model of all the users""" - - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - provider = models.ForeignKey(Connectors, on_delete=models.PROTECT, related_name="provider") - consumer = models.ForeignKey(Connectors, on_delete=models.PROTECT, related_name="consumer", null=True) - connector_pair_status = models.CharField(max_length=255, default="awaiting for approval") - ports = models.JSONField(default={}) - status = models.BooleanField(default=True) +# TODO - UNCOMMENT MODEL CURRENTLY COMMENTED TO AVOID CIRCULAR IMPORT +# @auto_str +# class ConnectorsMap(TimeStampMixin): +# """ConnectorsMap model of all the users""" +# +# id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) +# provider = models.ForeignKey(Connectors, on_delete=models.PROTECT, related_name="provider") +# consumer = models.ForeignKey(Connectors, on_delete=models.PROTECT, related_name="consumer", null=True) +# connector_pair_status = models.CharField(max_length=255, default="awaiting for approval") +# ports = models.JSONField(default={}) +# status = models.BooleanField(default=True) @auto_str @@ -149,7 +235,7 @@ class Resolution(TimeStampMixin): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) ticket = models.ForeignKey(SupportTicketV2, on_delete=models.CASCADE) # reference FK to the resolution providing authority - user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.CASCADE,null=True) + user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.CASCADE, null=True) resolution_text = models.CharField(max_length=1000) solution_attachments = models.FileField( upload_to=settings.RESOLUTIONS_ATTACHMENT_URL, @@ -160,3 +246,350 @@ class Resolution(TimeStampMixin): class Meta: db_table = "resolution" + + +@auto_str +class DatahubDocuments(models.Model): + """OrganizationDocuments model""" + + governing_law = models.TextField() + warranty = models.TextField() + limitations_of_liabilities = models.TextField() + privacy_policy = models.TextField() + tos = models.TextField() + + +# +# CATEGORY = ( +# ("crop_data", "crop_data"), +# ("partice_data", "partice_data"), +# ("farmer_profile", "farmer_profile"), +# ("land_records", "land_records"), +# ("research_data", "research_data"), +# ("cultivation_data", "cultivation_data"), +# ("soil_data", "soil_data"), +# ("weather_data", "weather_data"), +# ) + + +@auto_str +class StandardisationTemplate(TimeStampMixin): + """ + Data Standardisation Model. + datapoint category - Name of the category for a group of attributes + datapoint attribute - datapoints for each attribute (JSON) + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + datapoint_category = models.CharField(max_length=50, unique=True) + datapoint_description = models.TextField(max_length=255) + datapoint_attributes = models.JSONField(default=dict) + + +class Policy(TimeStampMixin): + """ + Policy documentation Model. + name - Name of the Policy. + description - datapoints of each Policy. + file - file of each policy. + """ + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=100, unique=True) + description = models.CharField(max_length=2048, unique=False) + file = models.FileField( + upload_to=settings.POLICY_FILES_URL, + validators=[validate_25MB_file_size], + null=True + ) + + +class CustomStorage(Storage): + def __init__(self, dataset_name, source): + self.dataset_name = dataset_name + self.source = source + + # def size(self, name): + # path = self.path(name) + # return os.path.getsize(path) + + def exists(self, name): + """ + Check if a file with the given name already exists in the storage. + """ + return os.path.exists(name) + + def url(self, url): + return url + + def _save(self, name, content): + # Save file to a directory outside MEDIA_ROOT + full_path = os.path.join(settings.DATASET_FILES_URL, name) + directory = os.path.dirname(full_path) + if not self.exists(directory): + os.makedirs(directory) + with open(full_path, 'wb') as f: + f.write(content.read()) + return name + + +@auto_str +class DatasetV2(TimeStampMixin): + """ + Stores a single dataset entry, related to :model:`datahub_userorganizationmap` (UserOrganizationMap). + New version of model for dataset - DatasetV2 to store Meta data of Datasets. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=50, unique=True) + user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.PROTECT, related_name="user_org_map") + description = models.TextField(max_length=512, null=True, blank=True) + category = models.JSONField(default=dict) + geography = models.JSONField(default=dict) + data_capture_start = models.DateTimeField(null=True, blank=True) + data_capture_end = models.DateTimeField(null=True, blank=True) + constantly_update = models.BooleanField(default=False) + is_temp = models.BooleanField(default=True) + + class Meta: + indexes = [models.Index(fields=["name", "category"])] + + +@auto_str +class DatasetV2File(TimeStampMixin): + """ + Stores a single file (file paths/urls) entry for datasets with a reference to DatasetV2 instance. + related to :model:`datahub_datasetv2` (DatasetV2) + + `Source` (enum): Enum to store file type + `file`: dataset of file type + `mysql`: dataset of mysql connection + `postgresql`: dataset of postgresql connection + """ + + def dataset_directory_path(instance, filename): + # file will be uploaded to MEDIA_ROOT/user_/ + return f"{settings.DATASET_FILES_URL}/{instance.dataset.name}/{instance.source}/{filename}" + + def get_upload_path(instance, filename): + return f"{instance.dataset.name}/{instance.source}/{filename}" + + def save(self, *args, **kwargs): + # set the user_id before saving + storage = CustomStorage(self.dataset.name, self.source) + self.file.storage = storage # type: ignore + + # if self.file: + # # Get the file size + # size = self.file.size + # self.file_size = size + + super().save(*args, **kwargs) + + SOURCES = [ + (Constants.SOURCE_FILE_TYPE, Constants.SOURCE_FILE_TYPE), + (Constants.SOURCE_MYSQL_FILE_TYPE, Constants.SOURCE_MYSQL_FILE_TYPE), + (Constants.SOURCE_POSTGRESQL_FILE_TYPE, Constants.SOURCE_POSTGRESQL_FILE_TYPE), + (Constants.SOURCE_API_TYPE, Constants.SOURCE_API_TYPE), + ] + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + dataset = models.ForeignKey(DatasetV2, on_delete=models.CASCADE, related_name="datasets") + file = models.FileField(upload_to=get_upload_path, null=True, blank=True) + file_size = models.PositiveIntegerField(null=True, blank=True) + source = models.CharField(max_length=50, choices=SOURCES) + standardised_file = models.FileField(upload_to=get_upload_path, null=True, blank=True) + standardised_configuration = models.JSONField(default=dict) + accessibility = models.CharField(max_length=255, null=True, choices=USAGE_POLICY_APPROVAL_STATUS, default="public") + connection_details = models.JSONField(default=dict, null=True) + + +class UsagePolicy(TimeStampMixin): + """ + Policy documentation Model. + datapoint category - Name of the category for a group of attributes + datapoint attribute - datapoints for each attribute (JSON) + """ + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user_organization_map = models.ForeignKey(UserOrganizationMap, on_delete=models.PROTECT, related_name="org") + dataset_file = models.ForeignKey(DatasetV2File, on_delete=models.CASCADE, related_name="dataset_v2_file") + approval_status = models.CharField(max_length=255, null=True, choices=USAGE_POLICY_REQUEST_STATUS, + default="requested") + accessibility_time = models.DateField(null=True) + type = models.CharField(max_length=20, null=True, choices=USAGE_POLICY_API_TYPE, default="dataset_file") + api_key = models.CharField(max_length=64, null=True, unique=True) + configs = models.JSONField(default=dict, null=True) + + +class Resource(TimeStampMixin): + """ + Resource Module -- Any user can create resource. + """ + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + title = models.CharField(max_length=100) + description = models.TextField(max_length=250) + user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.CASCADE) + category = models.JSONField(default=dict) + accessibility = models.CharField(max_length=255, null=True, choices=USAGE_POLICY_APPROVAL_STATUS, default="public") + + def __str__(self) -> str: + return self.title + + +class ResourceFile(TimeStampMixin): + """ + Resource Files Model -- Has a one to many relation + -- 1 resource can have multiple resource files. + """ + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + resource = models.ForeignKey(Resource, on_delete=models.CASCADE, related_name="resources") + file = models.FileField(upload_to=settings.RESOURCES_URL, null=True, blank=True) + file_size = models.PositiveIntegerField(null=True, blank=True) + type = models.CharField(max_length=20, null=True, choices=RESOURCE_URL_TYPE, default="file") + url = models.CharField(max_length=200, null=True) + transcription = models.CharField(max_length=10000, null=True, blank=True) + embeddings_status = models.CharField(max_length=20, null=True, choices=EMBEDDINGS_STATUS, default="in-progress") + embeddings_status_reason = models.CharField(max_length=1000, null=True) + + def __str__(self) -> str: + return self.file.name + + +class DatasetV2FileReload(TimeStampMixin): + dataset_file = models.ForeignKey(DatasetV2File, on_delete=models.CASCADE, related_name="dataset_file") + + +class Category(TimeStampMixin): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=50) + description = models.CharField(max_length=500) + + +class SubCategory(TimeStampMixin): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=100) + category = models.ForeignKey(Category, on_delete=models.CASCADE, related_name="subcategory_category") + + +class DatasetSubCategoryMap(TimeStampMixin): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + sub_category = models.ForeignKey(SubCategory, on_delete=models.CASCADE, related_name="dataset_sub_category_map") + dataset = models.ForeignKey(DatasetV2, on_delete=models.CASCADE, related_name="dataset_cat_map") + + +class ResourceSubCategoryMap(TimeStampMixin): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + sub_category = models.ForeignKey(SubCategory, on_delete=models.CASCADE, related_name="resource_sub_category_map") + resource = models.ForeignKey(Resource, on_delete=models.CASCADE, related_name="resource_cat_map") + + +class ResourceUsagePolicy(TimeStampMixin): + """ + Resource Policy Model. + """ + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user_organization_map = models.ForeignKey(UserOrganizationMap, on_delete=models.PROTECT, + related_name="org_usage_policy") + resource = models.ForeignKey(Resource, on_delete=models.CASCADE, related_name="resource_usage_policy") + type = models.CharField(max_length=200, null=True, choices=RESOURCE_USAGE_POLICY_API_TYPE, default="resource") + approval_status = models.CharField(max_length=255, null=True, choices=USAGE_POLICY_REQUEST_STATUS, + default="requested") + accessibility_time = models.DateField(null=True) + api_key = models.CharField(max_length=64, null=True, unique=True) + configs = models.JSONField(default=dict, null=True) + + +class LangchainPgCollection(models.Model): + name = models.CharField(max_length=50) + cmetadata = models.JSONField() + uuid = models.UUIDField(primary_key=True) + + # resource_file = models.ForeignKey(ResourceFile, on_delete=models.PROTECT, related_name="resource_file_collections") + + class Meta: + db_table = 'langchain_pg_collection' + + +class LangchainPgEmbedding(models.Model): + collection = models.ForeignKey(LangchainPgCollection, on_delete=models.CASCADE) + embedding = VectorField(1563) # Assuming 'vector' is a custom PostgreSQL data type + document = models.TextField() + cmetadata = models.JSONField() + custom_id = models.CharField(max_length=255) + uuid = models.UUIDField(primary_key=True) + + class Meta: + db_table = 'langchain_pg_embedding' + + # def __str__(self): + # return f"LangchainPgEmbedding(uuid={self.uuid}, document={self.document})" + + +# class Conversation(TimeStampMixin): +# BOT_CHOICES = ( +# ("whatsapp", "whatsapp"), +# ("telegram", "telegram"), +# ("vistaar", "vistaar"), + +# ) +# id = models.UUIDField(primary_key=True, default=uuid.uuid4, max_length=50) +# user = models.ForeignKey(UserOrganizationMap, on_delete=models.CASCADE, related_name="conversation_map") +# bot_type = models.CharField(max_length=10, choices=BOT_CHOICES, default="vistaar") +# bot_reference = models.CharField(max_length=50, null=True) +# # language = models.ForeignKeyField(Language, backref="language", null=True) +# # country = models.ForeignKeyField(Country, backref="country", null=True) +# # state = models.ForeignKeyField(State, backref="state", null=True) +# # crop = models.ForeignKeyField(Crop, backref="crop", null=True) + +# # class Meta: +# # table_name = "conversation" + +class Messages(TimeStampMixin): + INPUT_TYPES = ( + ("text", "text"), + ("voice", "voice"), + ) + BOT_CHOICES = ( + ("whatsapp", "whatsapp"), + ("telegram", "telegram"), + ("vistaar", "vistaar"), + ("vistaar_api", "vistaar_api"), + ) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, max_length=50) + user_map = models.ForeignKey(UserOrganizationMap, on_delete=models.CASCADE, related_name="conversation_user_map") + resource = models.ForeignKey(Resource, on_delete=models.CASCADE, related_name="messages_resource", null=True) + bot_type = models.CharField(max_length=20, choices=BOT_CHOICES, default="vistaar") + bot_reference = models.CharField(max_length=50, null=True) + query = models.CharField(max_length=10000, null=True) + translated_message = models.CharField(max_length=10000, null=True) + # message_input_time = models.DateTimeField(null=True) + # input_speech_to_text_start_time = models.DateTimeField(null=True) + # input_speech_to_text_end_time = models.DateTimeField(null=True) + # input_translation_start_time = models.DateTimeField(null=True) + # input_translation_end_time = models.DateTimeField(null=True) + query_response = models.CharField(max_length=10000, null=True) + # message_translated_response = models.CharField(max_length=10000, null=True) + # response_translation_start_time = models.DateTimeField(null=True) + # response_translation_end_time = models.DateTimeField(null=True) + # response_text_to_speech_start_time = models.DateTimeField(null=True) + # response_text_to_speech_end_time = models.DateTimeField(null=True) + # message_response_time = models.DateTimeField(null=True) + # main_bot_logic_start_time = models.DateTimeField(null=True) + # main_bot_logic_end_time = models.DateTimeField(null=True) + # video_retrieval_start_time = models.DateTimeField(null=True) + # video_retrieval_end_time = models.DateTimeField(null=True) + feedback = models.CharField(max_length=4096, null=True) + input_type = models.CharField(max_length=20, choices=INPUT_TYPES, null=True) + input_language_detected = models.CharField(max_length=20, null=True) + retrieved_chunks = models.ManyToManyField(LangchainPgEmbedding, null=True) + condensed_question = models.CharField(max_length=20000, null=True) + prompt_usage = models.JSONField(default={}, null=True) + + # class Meta: + # table_name = "messages" + +# class MessageEmbedding(models.Model): +# message = models.ForeignKey(Messages, on_delete=models.CASCADE) +# embedding = models.ForeignKey(LangchainPgEmbedding, on_delete=models.CASCADE) +# similarity = models.FloatField() + +# class Meta: +# db_table = 'datahub_messages_retrieved_chunks' \ No newline at end of file diff --git a/participant/serializers.py b/participant/serializers.py index 74ca10d0..5e70155f 100644 --- a/participant/serializers.py +++ b/participant/serializers.py @@ -1,28 +1,51 @@ import datetime import json +import os import re +import shutil +import string +from urllib.parse import quote +import logging +import plazy +import secrets from django.core.exceptions import ValidationError +from django.core.files.base import ContentFile +from django.db.models import Count, Prefetch, Q from django.utils.translation import gettext as _ -from rest_framework import serializers +from rest_framework import serializers, status from accounts import models -from accounts.models import User +from accounts.models import User, UserRole from accounts.serializers import UserSerializer +from core import settings from core.constants import Constants -from datahub.models import Datasets, Organization, UserOrganizationMap -from datahub.serializers import ( - OrganizationRetriveSerializer, - UserOrganizationMapSerializer, OrganizationSerializer, -) +from participant.models import Organization, UserOrganizationMap, UsagePolicy, ResourceFile, \ + LangchainPgEmbedding, Messages, DatasetV2, DatahubDocuments, DatasetSubCategoryMap, Category, SubCategory, \ + DatasetV2File, StandardisationTemplate, Policy, ResourceUsagePolicy, LangchainPgCollection, Resource, \ + ResourceSubCategoryMap + + +# TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT from participant.models import ( - Connectors, - ConnectorsMap, + # Connectors, + # ConnectorsMap, Department, Project, SupportTicket, SupportTicketV2, Resolution, ) + +from datasets.models import Datasets from utils import string_functions +from utils.custom_exceptions import NotFoundException +from utils.embeddings_creation import VectorDBBuilder +from utils.file_operations import move_directory, create_directory +from utils.string_functions import check_special_chars +from utils.validators import validate_file_size, validate_document_type, validate_image_type, validate_dataset_size, \ + validate_dataset_type +from utils.youtube_helper import get_youtube_url + +LOGGER = logging.getLogger(__name__) class TicketSupportSerializer(serializers.ModelSerializer): @@ -39,6 +62,20 @@ class Meta: fields = Constants.ALL +class OrganizationRetriveSerializer(serializers.ModelSerializer): + """_summary_ + + Args: + serializers (_type_): _description_ + """ + + class Meta: + """_summary_""" + + model = Organization + exclude = Constants.EXCLUDE_DATES + + class ParticipantSupportTicketSerializer(serializers.ModelSerializer): user_id = serializers.PrimaryKeyRelatedField( queryset=models.User.objects.all(), required=True, source="user_map.user" @@ -278,10 +315,12 @@ class Meta: ] -class ConnectorsSerializer(serializers.ModelSerializer): - class Meta: - model = Connectors - fields = Constants.ALL + +# TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT +# class ConnectorsSerializer(serializers.ModelSerializer): +# class Meta: +# # model = Connectors +# fields = Constants.ALL class ConnectorsSerializerForEmail(serializers.ModelSerializer): @@ -332,7 +371,9 @@ def to_representation(self, instance): dataset_detail = DatasetSerializer(required=False, read_only=True, source="dataset") class Meta: - model = Connectors + + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # # model = Connectors fields = [ "connector_name", "connector_type", @@ -348,7 +389,8 @@ class ConnectorsListSerializer(serializers.ModelSerializer): project_details = ProjectSerializer(required=False, allow_null=True, read_only=True, source="project") class Meta: - model = Connectors + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # model = Connectors # exclude = Constants.EXCLUDE_DATES fields = Constants.ALL @@ -378,7 +420,8 @@ class Meta: ) class Meta: - model = Connectors + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # model = Connectors # exclude = Constants.EXCLUDE_DATES fields = Constants.ALL @@ -411,10 +454,13 @@ class Meta: required=False, source="provider.user_map.user", ) - connector_details = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="provider") + + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # connector_details = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="provider") class Meta: - model = ConnectorsMap + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # model = ConnectorsMap # exclude = Constants.EXCLUDE_DATES fields = Constants.ALL @@ -447,35 +493,42 @@ class Meta: required=False, source="consumer.user_map.user", ) - connector_details = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="consumer") + + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # connector_details = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="consumer") class Meta: - model = ConnectorsMap + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # model = ConnectorsMap # exclude = Constants.EXCLUDE_DATES fields = Constants.ALL class ConnectorsConsumerRelationSerializer(serializers.ModelSerializer): - connectors = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="consumer") + # connectors = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="consumer") class Meta: - model = ConnectorsMap + # model = ConnectorsMap # exclude = Constants.EXCLUDE_DATES fields = Constants.ALL class ConnectorsProviderRelationSerializer(serializers.ModelSerializer): - connectors = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="provider") + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # connectors = ConnectorsSerializer(required=False, allow_null=True, read_only=True, source="provider") class Meta: - model = ConnectorsMap + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + # model = ConnectorsMap # exclude = Constants.EXCLUDE_DATES fields = Constants.ALL class ConnectorsMapSerializer(serializers.ModelSerializer): class Meta: - model = ConnectorsMap + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + + # model = ConnectorsMap # exclude = Constants.EXCLUDE_DATES fields = Constants.ALL @@ -488,7 +541,9 @@ class Meta: class ConnectorListSerializer(serializers.ModelSerializer): class Meta: - model = Connectors + # TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT + + # model = Connectors fields = ["id", "connector_name"] @@ -529,15 +584,13 @@ class DatabaseColumnRetrieveSerializer(serializers.Serializer): table_name = serializers.CharField(max_length=200, allow_blank=False) - - class DatabaseDataExportSerializer(serializers.Serializer): table_name = serializers.CharField(max_length=200, allow_blank=False) col = serializers.ListField(allow_empty=False) dataset_name = serializers.CharField(max_length=200, allow_blank=False) source = serializers.CharField(max_length=200, allow_blank=False) file_name = serializers.CharField(max_length=85, allow_blank=False) - filter_data = serializers.ListField(allow_empty=True,required=False) + filter_data = serializers.ListField(allow_empty=True, required=False) # { # "column_name" : "column_name", # "operation" : "operation", @@ -618,3 +671,1383 @@ class CreateSupportTicketResolutionsSerializer(serializers.ModelSerializer): class Meta: model = Resolution fields = '__all__' + + +# LOGGER = logging.getLogger(__name__) + + +# DATAHUB SERIALIZERS START FROM HERE > + + +class OrganizationSerializerExhaustive(serializers.ModelSerializer): + # org_email = serializers.EmailField() + # website = serializers.CharField() + + # def validate(self, attrs): + # # Use URLValidator to validate the website field + # website = attrs.get("website") + # if website: + # validator = URLValidator(schemes=["https"]) + # try: + # validator(website) + # except ValidationError: + # raise serializers.ValidationError({"website": "Invalid website URL"}) + + # return attrs + + class Meta: + model = Organization + + fields = "__all__" + + +class UserOrganizationCreateSerializer(serializers.Serializer): + """_summary_ + + Args: + serializers (_type_): _description_ + """ + + user = UserSerializer( + read_only=True, + allow_null=True, + required=False, + ) + organization = OrganizationRetriveSerializer( + read_only=True, + allow_null=True, + required=False, + ) + + # class Meta: + # """_summary_""" + + # model = UserOrganizationMap + # fields = [Constants.ORGANIZATION, Constants.USER] + # exclude = Constants.EXCLUDE_DATES + + +class UserOrganizationMapSerializer(serializers.ModelSerializer): + """_summary_ + + Args: + serializers (_type_): _description_ + """ + + class Meta: + """_summary_""" + + model = UserOrganizationMap + fields = Constants.ALL + # exclude = Constants.EXCLUDE_DATES + + +class ParticipantSerializer(serializers.ModelSerializer): + user_id = serializers.PrimaryKeyRelatedField( + queryset=models.User.objects.all(), + required=True, + source=Constants.USER, + ) + organization_id = serializers.PrimaryKeyRelatedField( + queryset=Organization.objects.all(), + allow_null=True, + required=False, + source=Constants.ORGANIZATION, + ) + user = UserSerializer( + read_only=False, + required=False, + ) + organization = OrganizationRetriveSerializer( + required=False, + allow_null=True, + read_only=True, + ) + + class Meta: + model = UserOrganizationMap + exclude = Constants.EXCLUDE_DATES + + dataset_count = serializers.SerializerMethodField(method_name="get_dataset_count") + content_files_count = serializers.SerializerMethodField(method_name="get_content_files_count") + connector_count = serializers.SerializerMethodField(method_name="get_connector_count") + number_of_participants = serializers.SerializerMethodField() + + def get_dataset_count(self, user_org_map): + return DatasetV2.objects.filter(user_map_id=user_org_map.id, is_temp=False).count() + + def get_content_files_count(self, user_org_map): + return ResourceFile.objects.select_related("resource").filter( + Q(resource__user_map_id=user_org_map.id) | + Q(resource__user_map__user__on_boarded_by=user_org_map.user_id) + ).values('type').annotate(count=Count('type')) + + def get_connector_count(self, user_org_map): + return Connectors.objects.filter(user_map_id=user_org_map.id).count() + + def get_number_of_participants(self, user_org_map): + return ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter(user__status=True, user__on_boarded_by=user_org_map.user.id, user__role=3) + .all() + .count() + ) + + +class DropDocumentSerializer(serializers.Serializer): + """DropDocumentSerializer class""" + + governing_law = serializers.FileField(validators=[validate_file_size, validate_document_type]) + privacy_policy = serializers.FileField(validators=[validate_file_size, validate_document_type]) + tos = serializers.FileField(validators=[validate_file_size, validate_document_type]) + limitations_of_liabilities = serializers.FileField(validators=[validate_file_size, validate_document_type]) + warranty = serializers.FileField(validators=[validate_file_size, validate_document_type]) + + +class PolicyDocumentSerializer(serializers.ModelSerializer): + """PolicyDocumentSerializer class""" + + governing_law = serializers.CharField() + privacy_policy = serializers.CharField() + tos = serializers.CharField() + limitations_of_liabilities = serializers.CharField() + warranty = serializers.CharField() + + class Meta: + model = DatahubDocuments + fields = Constants.ALL + + +class DatahubThemeSerializer(serializers.Serializer): + """DatahubThemeSerializer class""" + + banner = serializers.ImageField( + validators=[validate_file_size, validate_image_type], + required=False, + allow_null=True, + ) + button_color = serializers.CharField(required=False, allow_null=True) + # email = serializers.EmailField() + + +class TeamMemberListSerializer(serializers.Serializer): + """ + Create Team Member Serializer. + """ + + class Meta: + model = User + + id = serializers.UUIDField() + email = serializers.EmailField() + first_name = serializers.CharField() + last_name = serializers.CharField() + role = serializers.PrimaryKeyRelatedField(queryset=UserRole.objects.all(), read_only=False) + profile_picture = serializers.FileField() + status = serializers.BooleanField() + on_boarded = serializers.BooleanField() + + +class TeamMemberCreateSerializer(serializers.ModelSerializer): + """ + Create a Team Member + """ + + class Meta: + model = User + fields = ("email", "first_name", "last_name", "role", "on_boarded_by", "on_boarded") + + +class TeamMemberDetailsSerializer(serializers.ModelSerializer): + """ + Details of a Team Member + """ + + class Meta: + model = User + fields = ("id", "email", "first_name", "last_name", "role", "on_boarded_by", "on_boarded") + + +class TeamMemberUpdateSerializer(serializers.ModelSerializer): + """ + Update Team Member + """ + + class Meta: + model = User + fields = ("id", "email", "first_name", "last_name", "role", "on_boarded_by", "on_boarded") + + +# class DatasetSerializer(serializers.ModelSerializer): +# """_summary_ +# +# Args: +# serializers (_type_): _description_ +# """ +# +# def validate_sample_dataset(self, value): +# """ +# Validator function to check the file size limit. +# """ +# MAX_FILE_SIZE = ( +# Constants.MAX_PUBLIC_FILE_SIZE if self.initial_data.get("is_public") else Constants.MAX_FILE_SIZE +# ) +# filesize = value.size +# if filesize > MAX_FILE_SIZE: +# raise ValidationError( +# _("You cannot upload a file more than %(value)s MB"), +# params={"value": MAX_FILE_SIZE / 1048576}, +# ) +# return value +# +# class Meta: +# """_summary_""" +# +# model = Datasets +# fields = [ +# "user_map", +# "name", +# "description", +# "category", +# "geography", +# "crop_detail", +# "constantly_update", +# "dataset_size", +# "connector_availability", +# "age_of_date", +# "sample_dataset", +# "data_capture_start", +# "data_capture_end", +# "remarks", +# "is_enabled", +# "approval_status", +# "is_public", +# ] + + +class DatasetUpdateSerializer(serializers.ModelSerializer): + """_summary_ + + Args: + serializers (_type_): _description_ + """ + + class Meta: + """_summary_""" + + model = Datasets + fields = Constants.ALL + + +class DatahubDatasetsDetailSerializer(serializers.ModelSerializer): + user_id = serializers.PrimaryKeyRelatedField( + queryset=models.User.objects.all(), required=True, source="user_map.user" + ) + organization_id = serializers.PrimaryKeyRelatedField( + queryset=Organization.objects.all(), + allow_null=True, + required=False, + source="user_map.organization", + ) + user = UserSerializer( + read_only=False, + required=False, + allow_null=True, + source="user_map.user", + ) + organization = OrganizationRetriveSerializer( + required=False, allow_null=True, read_only=True, source="user_map.organization" + ) + + class Meta: + model = Datasets + exclude = Constants.EXCLUDE_DATES + + +class DatahubDatasetsSerializer(serializers.ModelSerializer): + class OrganizationDatsetsListRetriveSerializer(serializers.ModelSerializer): + class Meta: + model = Organization + fields = [ + "org_email", + "org_description", + "name", + "logo", + "address", + "phone_number", + ] + + class UserDatasetSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ["last_name", "first_name", "email", "on_boarded_by"] + + user_id = serializers.PrimaryKeyRelatedField( + queryset=models.User.objects.all(), required=True, source="user_map.user" + ) + organization_id = serializers.PrimaryKeyRelatedField( + queryset=Organization.objects.all(), + allow_null=True, + required=False, + source="user_map.organization", + ) + + organization = OrganizationDatsetsListRetriveSerializer( + required=False, allow_null=True, read_only=True, source="user_map.organization" + ) + user = UserDatasetSerializer(required=False, allow_null=True, read_only=True, source="user_map.user") + + class Meta: + model = Datasets + fields = Constants.ALL + + +class RecentSupportTicketSerializer(serializers.ModelSerializer): + class OrganizationRetriveSerializer(serializers.ModelSerializer): + class Meta: + model = Organization + fields = ["id", "org_email", "name"] + + class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ["id", "first_name", "last_name", "email", "role", "on_boarded_by"] + + organization = OrganizationRetriveSerializer( + allow_null=True, required=False, read_only=True, source="user_map.organization" + ) + + user = UserSerializer(allow_null=True, required=False, read_only=True, source="user_map.user") + + class Meta: + model = SupportTicket + fields = ["id", "subject", "category", "updated_at", "organization", "user"] + + +# class RecentConnectorListSerializer(serializers.ModelSerializer): +# class Meta: +# # model = Connectors +# fields = ["id", "connector_name", "updated_at", "dataset_count", "activity"] +# +# dataset_count = serializers.SerializerMethodField(method_name="get_dataset_count") +# activity = serializers.SerializerMethodField(method_name="get_activity") +# +# def get_dataset_count(self, connectors_queryset): +# return Datasets.objects.filter(status=True, user_map__user=connectors_queryset.user_map.user_id).count() +# +# def get_activity(self, connectors_queryset): +# try: +# if Connectors.objects.filter(status=True, user_map__id=connectors_queryset.user_map.id).first().status == True: +# return Constants.ACTIVE +# else: +# return Constants.NOT_ACTIVE +# except Exception as error: +# LOGGER.error(error, exc_info=True) +# +# return None + + +class RecentDatasetListSerializer(serializers.ModelSerializer): + class Meta: + model = Datasets + fields = ["id", "name", "updated_at", "connector_count", "activity"] + + connector_count = serializers.SerializerMethodField(method_name="get_connector_count") + activity = serializers.SerializerMethodField(method_name="get_activity") + + def get_connector_count(self, datasets_queryset): + return Connectors.objects.filter(status=True, dataset_id=datasets_queryset.id).count() + + def get_activity(self, datasets_queryset): + try: + datasets_queryset = Datasets.objects.filter(status=True, id=datasets_queryset.id) + if datasets_queryset: + if datasets_queryset.first().status == True: + return Constants.ACTIVE + else: + return Constants.NOT_ACTIVE + else: + return Constants.NOT_ACTIVE + except Exception as error: + LOGGER.error(error, exc_info=True) + + return None + + +class DatasetV2Validation(serializers.Serializer): + """ + Serializer to validate dataset name & dataset description. + """ + + def validate_dataset_name(self, name): + """ + Validator function to check if the dataset name includes special characters. + + **Parameters** + ``name`` (str): dataset name to validate + """ + if check_special_chars(name): + raise ValidationError("dataset name cannot include special characters.") + name = re.sub(r"\s+", " ", name) + + if not self.context.get("dataset_exists") and self.context.get("queryset"): + queryset = self.context.get("queryset") + if queryset.filter(name=name).exists(): + raise ValidationError("dataset v2 with this name already exists.") + + return name + + dataset_name = serializers.CharField(max_length=100, allow_null=False) + description = serializers.CharField(max_length=512, allow_null=False) + + +class DatasetV2TempFileSerializer(serializers.Serializer): + """ + Serializer for DatasetV2File model to serialize dataset files. + Following are the fields required by the serializer: + `datasets` (Files, mandatory): Multi upload Dataset files + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + """Remove fields based on the request type""" + if "context" in kwargs: + if "request_method" in kwargs["context"]: + request_method = kwargs.get("context").get("request_method") + if request_method == "DELETE" and not kwargs.get("context").get("query_params"): + # remove `datasets` field as `DELETE` method only requires `dataset_name`, `file_name` & `source` fields + self.fields.pop("datasets") + elif request_method == "DELETE" and kwargs.get("context").get("query_params"): + # remove `datasets` & `file_name` fields as `DELETE` method to delete directory only requires `dataset_name` & `source` field + self.fields.pop("datasets") + self.fields.pop("file_name") + self.fields.pop("source") + elif request_method == "POST": + # remove `file_name` field as `POST` method only requires `dataset_name`, `datasets` & `source` fields + self.fields.pop("file_name") + + def validate_datasets(self, files): + """ + Validator function to check for dataset file types & dataset file size (Constants.DATASET_MAX_FILE_SIZE) in MB. + + **Parameters** + ``files`` ([Files]): list of files to validate the file type included in Constants.DATASET_FILE_TYPES. + """ + for file in files: + if not validate_dataset_type(file, Constants.DATASET_FILE_TYPES): + raise ValidationError( + f"Document type not supported. Only following documents are allowed: {Constants.DATASET_FILE_TYPES}" + ) + + if not validate_dataset_size(file, Constants.DATASET_MAX_FILE_SIZE): + raise ValidationError( + f"You cannot upload/export file size more than {Constants.DATASET_MAX_FILE_SIZE}MB." + ) + + return files + + def validate_dataset_name(self, name): + """ + Validator function to check if the dataset name includes special characters. + + **Parameters** + ``name`` (str): dataset name to validate + """ + if check_special_chars(name): + raise ValidationError("dataset name cannot include special characters.") + name = re.sub(r"\s+", " ", name) + + if not self.context.get("dataset_exists") and self.context.get("queryset"): + queryset = self.context.get("queryset") + if queryset.filter(name=name).exists(): + raise ValidationError("dataset v2 with this name already exists.") + + return name + + dataset_name = serializers.CharField(max_length=100, allow_null=False) + datasets = serializers.ListField( + child=serializers.FileField(max_length=255, use_url=False, allow_empty_file=False), + write_only=True, + ) + file_name = serializers.CharField(allow_null=False) + source = serializers.CharField(allow_null=False) + + +class DatasetV2FileSerializer(serializers.ModelSerializer): + """ + Serializer for DatasetV2File model to serialize dataset files. + Following are the fields required by the serializer: + `id` (int): auto-generated Identifier + `dataset` (DatasetV2, FK): DatasetV2 reference object + `file` (File, mandatory): Dataset file + """ + + class Meta: + model = DatasetV2File + fields = [ + "id", + "dataset", + "file", + "source", + "standardised_file", + "standardised_configuration", + "accessibility", + ] + + +# class CategoryReverseSerializer(serializers.ModelSerializer): + +# class Meta: +# model = Category +# fields = ["id", "name", "subcategories"] + +# class SubCategoryReverseSerializer(serializers.ModelSerializer): +# category = CategoryReverseSerializer(read_only=True, many=True) +# class Meta: +# model = SubCategory +# fields = ["id", "name", "category"] + +class DatasetSubCategoryMapSerializer(serializers.ModelSerializer): + class Meta: + model = DatasetSubCategoryMap + fields = ['id', 'sub_category', 'dataset'] + + +class DatasetV2Serializer(serializers.ModelSerializer): + """ + Serializer for DatasetV2 model to serialize the Meta Data of Datasets. + Following are the fields required by the serializer: + `id` (UUID): auto-generated Identifier + `name` (String, unique, mandatory): Dataset name + `user_map` (UUID, mandatory): User Organization map ID, related to :model:`datahub_userorganizationmap` (UserOrganizationMap) + `description` (Text): Dataset description + `category` (JSON, mandatory): Category as JSON object + `geography` (String): Geography of the dataset + `data_capture_start` (DateTime): Start DateTime of the dataset captured + `data_capture_end` (DateTime): End DateTime of the dataset captured + `datasets` (Files, FK, read only): Dataset files stored + `upload_datasets` (List, mandatory): List of dataset files to be uploaded + """ + + class OrganizationRetriveSerializer(serializers.ModelSerializer): + class Meta: + model = Organization + fields = [ + "org_email", + "org_description", + "name", + "logo", + "phone_number", + "address", + ] + + class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ["id", "first_name", "last_name", "email", "on_boarded_by"] + + organization = OrganizationRetriveSerializer( + allow_null=True, required=False, read_only=True, source="user_map.organization" + ) + categories = serializers.SerializerMethodField() + + user = UserSerializer(allow_null=True, required=False, read_only=True, source="user_map.user") + + datasets = DatasetV2FileSerializer(many=True, read_only=True) + upload_datasets = serializers.ListField( + child=serializers.FileField(max_length=255, use_url=False, allow_empty_file=False), + write_only=True, + required=False, + ) + + def validate_name(self, name): + if check_special_chars(name): + raise ValidationError("dataset name cannot include special characters.") + name = re.sub(r"\s+", " ", name) + return name + + class Meta: + model = DatasetV2 + fields = [ + "id", + "name", + "user_map", + "description", + "category", + "geography", + "constantly_update", + "data_capture_start", + "data_capture_end", + "organization", + "user", + "datasets", + "upload_datasets", + "categories" + ] + + def get_categories(self, instance): + category_and_sub_category = Category.objects.prefetch_related( + Prefetch("subcategory_category", + queryset=SubCategory.objects.prefetch_related( + "dataset_sub_category_map__dataset").filter( + dataset_sub_category_map__dataset_id=instance.id), + ), + 'subcategory_category__dataset_sub_category_map' + ).filter(subcategory_category__dataset_sub_category_map__dataset_id=instance.id).distinct().all() + serializer = CategorySerializer(category_and_sub_category, many=True) + return serializer.data + + def create(self, validated_data): + """ + Override the create method to save meta data (DatasetV2) with multiple dataset files on to the referenced model (DatasetV2File). + + **Parameters** + ``validated_data`` (Dict): Validated data from the serializer + + **Returns** + ``dataset_obj`` (DatasetV2 instance): Save & return the dataset + """ + # create meta dataset obj + # uploaded_files = validated_data.pop("upload_datasets") + file_paths = {} + + try: + # create_directory() + directory_created = move_directory( + os.path.join(settings.BASE_DIR, settings.TEMP_DATASET_URL, validated_data.get("name")), + settings.DATASET_FILES_URL, + ) + to_find = [ + Constants.SOURCE_FILE_TYPE, + Constants.SOURCE_MYSQL_FILE_TYPE, + Constants.SOURCE_POSTGRESQL_FILE_TYPE, + ] + + # import pdb + # pdb.set_trace() + + standardised_directory_created = move_directory( + os.path.join(settings.BASE_DIR, settings.TEMP_STANDARDISED_DIR, validated_data.get("name")), + settings.STANDARDISED_FILES_URL, + ) + + file_paths = plazy.list_files(root=directory_created, is_include_root=True) + standardisation_template = json.loads(self.context.get("standardisation_template")) + standardisation_config = json.loads(self.context.get("standardisation_config", {})) + if file_paths: + dataset_obj = DatasetV2.objects.create(**validated_data) + for file_path in file_paths: + dataset_file_path = file_path.replace("media/", "") + dataset_name_file_path = "/".join(dataset_file_path.split("/")[-3:]) + DatasetV2File.objects.create( + dataset=dataset_obj, + file=dataset_file_path, + source=file_path.split("/")[-2], + standardised_file=(settings.STANDARDISED_FILES_URL + dataset_name_file_path).replace( + "media/", "" + ) + if os.path.isfile( + settings.STANDARDISED_FILES_URL + + standardisation_template.get("temp/datasets/" + dataset_name_file_path, "") + ) + else dataset_file_path, + standardised_configuration=standardisation_config.get( + "temp/datasets/" + dataset_name_file_path + ) + if standardisation_config.get("temp/datasets/" + dataset_name_file_path, "") + else {}, + ) + return dataset_obj + + except Exception as error: + LOGGER.error(error, exc_info=True) + raise NotFoundException( + detail="Dataset files are not uploaded or missing. Failed to create meta dataset.", + status_code=status.HTTP_400_BAD_REQUEST, + ) + + def update(self, instance, validated_data): + """ + Override the update method to save meta data (DatasetV2) with multiple new dataset files on to the referenced model (DatasetV2File). + + **Parameters** + ``instance`` (obj): Instance of DatasetV2 model + ``validated_data`` (Dict): Validated data from the serializer + + **Returns** + ``instance`` (DatasetV2 instance): Save & return the dataset + """ + temp_directory = os.path.join(settings.TEMP_DATASET_URL, instance.name) + file_paths = {} + to_find = [ + Constants.SOURCE_FILE_TYPE, + Constants.SOURCE_MYSQL_FILE_TYPE, + Constants.SOURCE_POSTGRESQL_FILE_TYPE, + ] + + # iterate through temp_directory to fetch file paths & file names + # for sub_dir in to_find: + # direct = os.path.join(temp_directory, sub_dir) + # if os.path.exists(direct): + # file_paths.update({ + # os.path.join(direct, f): [sub_dir, f] + # for f in os.listdir(direct) + # if os.path.isfile(os.path.join(direct, f)) + # }) + + # # save the files at actual dataset location & update in DatasetV2File table + # if file_paths: + # for file_path, sub_file in file_paths.items(): + # directory_created = create_directory(os.path.join(settings.DATASET_FILES_URL), [instance.name, sub_file[0]]) + # shutil.copy(file_path, directory_created) + + # path_to_save = os.path.join(directory_created, sub_file[1]) + # if not DatasetV2File.objects.filter(file=path_to_save.replace("media/", "")): + # DatasetV2File.objects.create(dataset=instance, file=path_to_save.replace("media/", ""), source=sub_file[0]) + + # save the files at actual dataset location & update in DatasetV2File table + # standardised_directory_created = move_directory( + # os.path.join(settings.TEMP_STANDARDISED_DIR,instance.name), settings.STANDARDISED_FILES_URL + # ) + standardised_temp_directory = os.path.join(settings.TEMP_STANDARDISED_DIR, instance.name) + standardised_file_paths = ( + plazy.list_files(root=standardised_temp_directory, is_include_root=True) + if os.path.exists(standardised_temp_directory) + else None + ) + standardisation_template = json.loads(self.context.get("standardisation_template")) + standardisation_config = json.loads(self.context.get("standardisation_config", {})) + + if standardised_file_paths: + for file_path in standardised_file_paths: + directory_created = create_directory( + os.path.join(settings.STANDARDISED_FILES_URL), [instance.name, file_path.split("/")[-2]] + ) + # file_path = file_path.replace("temp/standardised/","") + shutil.copy2(file_path, directory_created) + + dataset_name_file_path = str(directory_created).replace("media/", "") + file_path.split("/")[-1] + dataset_file_path_alone = "datasets/" + "/".join(file_path.split("/")[-3:]) + standardised_dataset_file_path_alone = "standardised/" + "/".join(file_path.split("/")[-3:]) + print("*****", dataset_name_file_path) + # import pdb; pdb.set_trace() + # path_to_save = os.path.join(directory_created, file_path.split("/")[-1]) + DatasetV2File.objects.filter(file=dataset_file_path_alone).update( + dataset=instance, + source=file_path.split("/")[-2], + standardised_file=standardised_dataset_file_path_alone, + standardised_configuration=standardisation_config.get( + str(directory_created) + file_path.split("/")[-1] + ) + if standardisation_config.get(str(directory_created) + file_path.split("/")[-1], "") + else {}, + ) + # delete the temp directory + shutil.rmtree(standardised_temp_directory) + + if os.path.exists(temp_directory): + file_paths = ( + plazy.list_files(root=temp_directory, is_include_root=True) if os.path.exists(temp_directory) else None + ) + + if file_paths: + for file_path in file_paths: + directory_created = create_directory( + os.path.join(settings.DATASET_FILES_URL), [instance.name, file_path.split("/")[-2]] + ) + shutil.copy2(file_path, directory_created) + dataset_file_path = file_path.replace("media/", "") + dataset_name_file_path = "/".join(dataset_file_path.split("/")[-3:]) + + path_to_save = os.path.join(directory_created, file_path.split("/")[-1]) + if not DatasetV2File.objects.filter(standardised_file=path_to_save.replace("media/", "")): + DatasetV2File.objects.create( + dataset=instance, + file=path_to_save.replace("media/", ""), + source=file_path.split("/")[-2], + standardised_file=(settings.STANDARDISED_FILES_URL + dataset_name_file_path).replace( + "media/", "" + ) + if os.path.isfile( + settings.STANDARDISED_FILES_URL + + standardisation_template.get("temp/datasets/" + dataset_name_file_path, "") + ) + else dataset_file_path, + standardised_configuration=standardisation_config.get( + "temp/datasets/" + dataset_name_file_path + ) + if standardisation_config.get("temp/datasets/" + dataset_name_file_path, "") + else {}, + ) + + # delete the temp directory + shutil.rmtree(temp_directory) + + instance = super(DatasetV2Serializer, self).update(instance, validated_data) + return instance + + +class DatahubDatasetsV2Serializer(serializers.ModelSerializer): + """ + Serializer for filtered list of datasets. + """ + + user_id = serializers.PrimaryKeyRelatedField( + queryset=models.User.objects.all(), required=True, source="user_map.user" + ) + organization_id = serializers.PrimaryKeyRelatedField( + queryset=Organization.objects.all(), + allow_null=True, + required=False, + source="user_map.organization", + ) + categories = serializers.SerializerMethodField() + organization = DatahubDatasetsSerializer.OrganizationDatsetsListRetriveSerializer( + required=False, allow_null=True, read_only=True, source="user_map.organization" + ) + user = DatahubDatasetsSerializer.UserDatasetSerializer( + required=False, allow_null=True, read_only=True, source="user_map.user" + ) + + class Meta: + model = DatasetV2 + fields = Constants.ALL + + def get_categories(self, instance): + category_and_sub_category = Category.objects.prefetch_related( + Prefetch("subcategory_category", + queryset=SubCategory.objects.prefetch_related( + "dataset_sub_category_map__dataset").filter( + dataset_sub_category_map__dataset_id=instance.id), + ), + 'subcategory_category__dataset_sub_category_map' + ).filter(subcategory_category__dataset_sub_category_map__dataset_id=instance.id).distinct().all() + serializer = CategorySerializer(category_and_sub_category, many=True) + return serializer.data + + +class micrositeOrganizationSerializer(serializers.ModelSerializer): + organization_id = serializers.PrimaryKeyRelatedField( + queryset=Organization.objects.all(), + allow_null=True, + required=False, + source=Constants.ORGANIZATION, + ) + organization = OrganizationRetriveSerializer( + required=False, + allow_null=True, + read_only=True, + ) + + class Meta: + model = UserOrganizationMap + exclude = Constants.EXCLUDE_DATES + + dataset_count = serializers.SerializerMethodField(method_name="get_dataset_count") + users_count = serializers.SerializerMethodField(method_name="get_users_count") + + def get_dataset_count(self, user_org_map): + return DatasetV2.objects.filter(user_map__organization=user_org_map.organization.id, is_temp=False).count() + + def get_users_count(self, user_org_map): + return UserOrganizationMap.objects.filter( + user__status=True, organization_id=user_org_map.organization.id + ).count() + + +class StandardisationTemplateViewSerializer(serializers.ModelSerializer): + class Meta: + model = StandardisationTemplate + exclude = Constants.EXCLUDE_DATES + + +class StandardisationTemplateUpdateSerializer(serializers.ModelSerializer): + class Meta: + model = StandardisationTemplate + exclude = Constants.EXCLUDE_DATES + + +class PolicySerializer(serializers.ModelSerializer): + class Meta: + model = Policy + fields = Constants.ALL + + +class DatasetV2NewListSerializer(serializers.ModelSerializer): + # dataset = DatasetFileV2NewSerializer() + class Meta: + model = DatasetV2 + fields = Constants.ALL + + +class DatasetFileV2NewSerializer(serializers.ModelSerializer): + class Meta: + model = DatasetV2File + exclude = ["standardised_file"] + + +class DatasetFileV2StandardisedSerializer(serializers.ModelSerializer): + class Meta: + model = DatasetV2File + fields = Constants.ALL + + +class DatasetFileV2ListSerializer(serializers.ModelSerializer): + class Meta: + model = DatasetV2File + exclude = ["created_at", "updated_at"] + + +class DatasetV2ListNewSerializer(serializers.ModelSerializer): + dataset_files = DatasetFileV2ListSerializer(many=True, source="datasets") + + class Meta: + model = DatasetV2 + exclude = ["created_at", "updated_at"] + + +class DatasetV2DetailNewSerializer(serializers.ModelSerializer): + dataset_files = DatasetFileV2StandardisedSerializer(many=True, source="datasets") + + class Meta: + model = DatasetV2 + fields = Constants.ALL + # fields = ['id', 'name', 'geography', 'category', 'dataset_files'] + + +class UsagePolicySerializer(serializers.ModelSerializer): + class Meta: + model = UsagePolicy + exclude = ["created_at", "updated_at", "approval_status"] + + +class UsageUpdatePolicySerializer(serializers.ModelSerializer): + class Meta: + model = UsagePolicy + fields = "__all__" + + +class APIBuilderSerializer(serializers.ModelSerializer): + class Meta: + model = UsagePolicy + fields = ["approval_status", "accessibility_time", "api_key"] + + +class UsagePolicyDetailSerializer(serializers.ModelSerializer): + organization = DatahubDatasetsSerializer.OrganizationDatsetsListRetriveSerializer( + required=False, allow_null=True, read_only=True, source="user_organization_map.organization" + ) + user = DatahubDatasetsSerializer.UserDatasetSerializer( + required=False, allow_null=True, read_only=True, source="user_organization_map.user" + ) + + class Meta: + model = UsagePolicy + fields = "__all__" + + +class ResourceUsagePolicyDetailSerializer(serializers.ModelSerializer): + organization = DatahubDatasetsSerializer.OrganizationDatsetsListRetriveSerializer( + required=False, allow_null=True, read_only=True, source="user_organization_map.organization" + ) + user = DatahubDatasetsSerializer.UserDatasetSerializer( + required=False, allow_null=True, read_only=True, source="user_organization_map.user" + ) + + class Meta: + model = ResourceUsagePolicy + fields = "__all__" + + +class CustomJSONField(serializers.JSONField): + """ + Custom JSON field to handle non-JSON data. + """ + + def to_representation(self, obj): + """ + Serialize the field value. + """ + try: + return super().to_representation(obj) + except Exception as e: + # Handle non-JSON data here, for example, by converting it to a JSON-compatible format + return str(obj) + + +class LangChainEmbeddingsSerializer(serializers.ModelSerializer): + # cmetadata = CustomJSONField() + class Meta: + model = LangchainPgEmbedding + fields = ["embedding", "document"] + + # def to_representation(self, instance): + # try: + # data = super().to_representation(instance) + # # Attempt to decode JSON in cmetadata + # data['cmetadata'] = json.loads(data['cmetadata']) + # return data + # except json.JSONDecodeError: + # # Handle the case where cmetadata is not valid JSON + # data['cmetadata'] = {} # Provide a default value or appropriate fallback + # return data + + +class ResourceFileSerializer(serializers.ModelSerializer): + # collections = serializers.SerializerMethodField() + class Meta: + model = ResourceFile + fields = "__all__" + + def get_collections(self, obj): + # Assuming that 'obj' is an instance of ResourceFile + # Retrieve the related LangchainPgEmbedding instances + collection = LangchainPgCollection.objects.filter(name=str(obj.id)).first() + # return collection + # print(collection) + # import pdb; pdb.set_trace() + if collection: + embeddings = LangchainPgEmbedding.objects.filter(collection_id=collection.uuid).values("embedding", + "document") + + # print(embeddings) + # # Serialize the retrieved embeddings using LangchainPgEmbeddingSerializert + # embeddings_serializer = LangChainEmbeddingsSerializer(embeddings, many=True) + + # Return the serialized embeddings + return embeddings + return [] + + +class DatahubDatasetFileDashboardFilterSerializer(serializers.Serializer): + county = serializers.ListField(allow_empty=False, required=True) + sub_county = serializers.ListField(allow_empty=False, required=False) + gender = serializers.ListField(allow_empty=False, required=False) + value_chain = serializers.ListField(allow_empty=False, required=False) + + +class SubCategorySerializer(serializers.ModelSerializer): + class Meta: + model = SubCategory + fields = ["id", "name", "category"] + + +class CategorySerializer(serializers.ModelSerializer): + subcategories = SubCategorySerializer(many=True, + read_only=True, source="subcategory_category" + ) + + class Meta: + model = Category + fields = ["id", "name", "subcategories"] + + +class ResourceSubCategoryMapSerializer(serializers.ModelSerializer): + sub_categories = CategorySerializer(many=True, read_only=True, source="subcategory_category") + + class Meta: + model = ResourceSubCategoryMap + fields = "__all__" + + +class ResourceSerializer(serializers.ModelSerializer): + class OrganizationRetriveSerializer(serializers.ModelSerializer): + class Meta: + model = Organization + fields = ["id", "org_email", "name", "logo", "address"] + + class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ["id", "first_name", "last_name", "email", "role", "on_boarded_by"] + + categories = serializers.SerializerMethodField(read_only=True) + resources = ResourceFileSerializer(many=True, read_only=True) + uploaded_files = serializers.ListField(child=serializers.JSONField(), write_only=True, required=False) + files = serializers.ListField(child=serializers.ListField(), write_only=True, required=False) + + sub_categories_map = serializers.ListField(write_only=True) + + organization = OrganizationRetriveSerializer( + allow_null=True, required=False, read_only=True, source="user_map.organization" + ) + user = UserSerializer(allow_null=True, required=False, read_only=True, source="user_map.user") + content_files_count = serializers.SerializerMethodField(method_name="get_content_files_count") + + class Meta: + model = Resource + fields = ( + "id", + "title", + "description", + "user_map", + "category", + "resources", + "uploaded_files", + "organization", + "user", + "created_at", + "updated_at", + "content_files_count", + "sub_categories_map", + "categories", + "files" + ) + + def get_categories(self, instance): + category_and_sub_category = Category.objects.prefetch_related( + Prefetch("subcategory_category", + queryset=SubCategory.objects.prefetch_related( + "resource_sub_category_map").filter( + resource_sub_category_map__resource=instance.id), + ), + 'subcategory_category__resource_sub_category_map' + ).filter(subcategory_category__resource_sub_category_map__resource=instance.id).distinct().all() + serializer = CategorySerializer(category_and_sub_category, many=True) + return serializer.data + + def get_content_files_count(self, resource): + return ResourceFile.objects.filter(resource=resource.id).values('type').annotate(count=Count('type')) + + def construct_file_path(self, instance, filename): + # Generate a unique string to append to the filename + unique_str = get_random_string(length=8) + # Construct the file path + file_path = f"users/resources/{unique_str}_{filename.replace('/', '')}" + return file_path + + def create(self, validated_data): + try: + resource_files_data = validated_data.pop("uploaded_files") + resource_files = validated_data.pop("files") + sub_categories_map = validated_data.pop("sub_categories_map") + resource = Resource.objects.create(**validated_data) + resource_files_data = json.loads(resource_files_data[0]) if resource_files_data else [] + sub_categories_map = json.loads(sub_categories_map[0]) if sub_categories_map else [] + import pdb; + pdb.set_trace() + resource_sub_cat_instances = [ + ResourceSubCategoryMap(resource=resource, sub_category=SubCategory.objects.get(id=sub_cat) + ) for sub_cat in sub_categories_map] + + ResourceSubCategoryMap.objects.bulk_create(resource_sub_cat_instances) + + for resource_file in resource_files_data: + if resource_file.get("type") == "youtube": + youtube_urls_response = get_youtube_url(resource_file.get("url")) + if youtube_urls_response.status_code == 400: + return youtube_urls_response + youtube_urls = youtube_urls_response.data + playlist_urls = [{"resource": resource.id, "type": "youtube", **row} for row in youtube_urls] + for row in playlist_urls: + serializer = ResourceFileSerializer(data=row, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + LOGGER.info(f"Embeding creation started for youtube url: {row.get('url')}") + VectorDBBuilder.create_vector_db.delay(serializer.data) + if resource_file.get("type") == "api": + with open(resource_file.get("file").replace("/media/", ''), + "rb") as outfile: # Open the file in binary read mode + # Wrap the file content using Django's ContentFile + django_file = ContentFile(outfile.read(), + name=f"{resource_file.get('file_name', 'file')}.json") # You can give it any name you prefer + # Prepare data for serializer + serializer_data = {"resource": resource.id, "type": "api", "file": django_file} + serializer = ResourceFileSerializer(data=serializer_data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + LOGGER.info(f"Embeding creation started for youtube url: {resource_file.get('file')}") + VectorDBBuilder.create_vector_db.delay(serializer.data) + else: + serializer = ResourceFileSerializer(data={"resource": resource.id, **resource_file}, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + LOGGER.info( + f"Embeding creation started for url: {resource_file.get('url')} or file: {resource_file.get('url')}") + VectorDBBuilder.create_vector_db.delay(serializer.data) + for file in resource_files[0]: + data = {"resource": resource.id, "file": file, "type": "file"} + serializer = ResourceFileSerializer(data=data) + serializer.is_valid(raise_exception=True) + serializer.save() + VectorDBBuilder.create_vector_db.delay(serializer.data) + return resource + except Exception as e: + LOGGER.error(e, exc_info=True) + return e + # def update(self, instance, validated_data): + # uploaded_files_data = validated_data.pop('uploaded_files', []) + + # for attr, value in validated_data.items(): + # setattr(instance, attr, value) + # instance.save() + + # # Handle existing files + # # import pdb; pdb.set_trace() + # existing_file_ids = [] + # for file_data in uploaded_files_data: + # file_id = file_data.get('id', None) + # if file_id and file_data.get('delete', None): # Existing file + # existing_file = ResourceFile.objects.get(id=file_id) + # existing_file_ids.append(existing_file.id) + # # Update file attributes if needed + # else: # New file + # ResourceFile.objects.create(resource=instance, file=file_data['file']) + + # # Handle deletion of files not present in uploaded_files_data + # unwanted_files = ResourceFile.objects.filter(resource=instance).exclude(id__in=existing_file_ids) + # unwanted_files.delete() + # return instance + + +# class ResourceFileSerializer(serializers.ModelSerializer): +# class Meta: +# model = ResourceFile +# fields = "__all__" + + +class ParticipantCostewardSerializer(serializers.ModelSerializer): + user_id = serializers.PrimaryKeyRelatedField( + queryset=models.User.objects.all(), + required=True, + source=Constants.USER, + ) + organization_id = serializers.PrimaryKeyRelatedField( + queryset=Organization.objects.all(), + allow_null=True, + required=False, + source=Constants.ORGANIZATION, + ) + user = UserSerializer( + read_only=False, + required=False, + ) + organization = OrganizationRetriveSerializer( + required=False, + allow_null=True, + read_only=True, + ) + + class Meta: + model = UserOrganizationMap + exclude = Constants.EXCLUDE_DATES + + dataset_count = serializers.SerializerMethodField(method_name="get_dataset_count") + content_files_count = serializers.SerializerMethodField(method_name="get_content_files_count") + connector_count = serializers.SerializerMethodField(method_name="get_connector_count") + number_of_participants = serializers.SerializerMethodField() + + def get_dataset_count(self, user_org_map): + return DatasetV2.objects.filter(user_map_id=user_org_map.id, is_temp=False).count() + + def get_content_files_count(self, user_org_map): + return ResourceFile.objects.select_related("resource").filter( + Q(resource__user_map_id=user_org_map.id) | + Q(resource__user_map__user__on_boarded_by=user_org_map.user_id) + ).values('type').annotate(count=Count('type')) + + def get_connector_count(self, user_org_map): + return Connectors.objects.filter(user_map_id=user_org_map.id).count() + + def get_number_of_participants(self, user_org_map): + return ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter(user__status=True, user__on_boarded_by=user_org_map.user.id, user__role=3) + .all() + .count() + ) + + +class ResourceUsagePolicySerializer(serializers.ModelSerializer): + class Meta: + model = ResourceUsagePolicy + fields = "__all__" + + +class ResourceAPIBuilderSerializer(serializers.ModelSerializer): + class Meta: + model = ResourceUsagePolicy + fields = ["approval_status", "accessibility_time", "api_key"] + + +def get_random_string(length=8): + characters = string.ascii_letters + string.digits + unique_str = ''.join(secrets.choice(characters) for _ in range(length)) + return quote(unique_str, safe='') + + +# class ConversationSerializer(serializers.ModelSerializer): +# class Meta: +# model = Conversation +# fields = "__all__" + +class MessagesSerializer(serializers.ModelSerializer): + class Meta: + model = Messages + fields = "__all__" + + +class MessagesRetriveSerializer(serializers.ModelSerializer): + class Meta: + model = Messages + exclude = ["retrieved_chunks"] + + +class MessagesChunksRetriveSerializer(serializers.ModelSerializer): + retrieved_chunks = serializers.SerializerMethodField() + + class Meta: + model = Messages + fields = "__all__" + + def get_retrieved_chunks(self, instance): + related_embeddings = instance.retrieved_chunks.defer("cmetadata").all() + related_documents = [embedding.document for embedding in related_embeddings] + return related_documents + + +class ResourceListSerializer(serializers.ModelSerializer): + class OrganizationRetriveSerializer(serializers.ModelSerializer): + class Meta: + model = Organization + fields = ["id", "org_email", "name", "logo", "address"] + + class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ["id", "first_name", "last_name", "email", "role", "on_boarded_by"] + + organization = OrganizationRetriveSerializer( + allow_null=True, required=False, read_only=True, source="user_map.organization" + ) + user = UserSerializer(allow_null=True, required=False, read_only=True, source="user_map.user") + content_files_count = serializers.SerializerMethodField(method_name="get_content_files_count") + + class Meta: + model = Resource + fields = "__all__" + + def get_content_files_count(self, resource): + return ResourceFile.objects.filter(resource=resource.id).values('type').annotate(count=Count('type')) + +class OrganizationSerializer(serializers.ModelSerializer): + # org_email = serializers.EmailField() + # website = serializers.CharField() + + # def validate(self, attrs): + # # Use URLValidator to validate the website field + # website = attrs.get("website") + # if website: + # validator = URLValidator(schemes=["https"]) + # try: + # validator(website) + # except ValidationError: + # raise serializers.ValidationError({"website": "Invalid website URL"}) + + # return attrs + + class Meta: + model = Organization + + fields="__all__" \ No newline at end of file diff --git a/participant/tests/test_create_new_req_views.py b/participant/tests/test_create_new_req_views.py index c65a1109..64ebf02b 100644 --- a/participant/tests/test_create_new_req_views.py +++ b/participant/tests/test_create_new_req_views.py @@ -2,7 +2,7 @@ from django.test import Client, TestCase from rest_framework import status import json -from datahub.models import Organization, UserOrganizationMap +from participant.models import Organization, UserOrganizationMap from accounts.models import User, UserRole from participant.models import SupportTicketV2 from rest_framework_simplejwt.tokens import RefreshToken diff --git a/participant/tests/test_support_ticket_views.py b/participant/tests/test_support_ticket_views.py index 6ad56e33..31e144c2 100644 --- a/participant/tests/test_support_ticket_views.py +++ b/participant/tests/test_support_ticket_views.py @@ -2,7 +2,7 @@ from django.test import Client, TestCase from rest_framework import status import json -from datahub.models import Organization, UserOrganizationMap +from participant.models import Organization, UserOrganizationMap from accounts.models import User, UserRole from participant.models import SupportTicketV2 from rest_framework_simplejwt.tokens import RefreshToken diff --git a/participant/tests/test_views.py b/participant/tests/test_views.py index c13c4a29..d49a7c93 100644 --- a/participant/tests/test_views.py +++ b/participant/tests/test_views.py @@ -6,10 +6,12 @@ from _pytest.monkeypatch import MonkeyPatch from accounts.models import User, UserRole -from datahub.models import Datasets, Organization, UserOrganizationMap +from participant.models import Organization, UserOrganizationMap +from datasets.models import Datasets from django.test import Client, TestCase from django.urls import reverse -from participant.models import Connectors, SupportTicket +# TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT +# from participant.models import Connectors, SupportTicket from requests import request from requests_toolbelt.multipart.encoder import MultipartEncoder from rest_framework import serializers diff --git a/participant/tests/test_views_support.py b/participant/tests/test_views_support.py index bf4590d7..2e943481 100644 --- a/participant/tests/test_views_support.py +++ b/participant/tests/test_views_support.py @@ -9,7 +9,7 @@ from rest_framework_simplejwt.tokens import RefreshToken from accounts.models import User, UserRole -from datahub.models import Organization, UserOrganizationMap +from participant.models import Organization, UserOrganizationMap from participant.models import Resolution, SupportTicketV2 diff --git a/participant/urls.py b/participant/urls.py index ca54db49..917d2ce6 100644 --- a/participant/urls.py +++ b/participant/urls.py @@ -13,6 +13,37 @@ SupportTicketV2ModelViewSet, SupportTicketResolutionsViewset +) +from participant.views import ( + CategoryViewSet, + DatahubDashboard, + DatahubNewDashboard, + DatahubThemeView, + DatasetFileV2View, + DatasetV2View, + DatasetV2ViewSet, + DatasetV2ViewSetOps, + DocumentSaveView, + DropDocumentView, + EmbeddingsViewSet, + MailInvitationViewSet, + MessagesCreateViewSet, + MessagesViewSet, + OrganizationViewSet, + ParticipantViewSet, + PolicyDetailAPIView, + PolicyListAPIView, + ResourceFileManagementViewSet, + ResourceManagementViewSet, + ResourceUsagePolicyListCreateView, + ResourceUsagePolicyRetrieveUpdateDestroyView, + StandardisationTemplateView, + SubCategoryViewSet, + SupportViewSet, + TeamMemberViewSet, + UsagePolicyListCreateView, + UsagePolicyRetrieveUpdateDestroyView, + ) router = DefaultRouter() @@ -25,10 +56,38 @@ router.register(r"database", DataBaseViewSet,basename="database") router.register(r"support_ticket", SupportTicketV2ModelViewSet,basename="support_tickets") router.register(r"ticket_resolution", SupportTicketResolutionsViewset,basename="support_tickets_resolutions") +router.register(r"participant", ParticipantViewSet, basename=Constants.PARTICIPANT) +router.register(r"send_invite", MailInvitationViewSet, basename=Constants.SEND_INVITE) +router.register(r"organization", OrganizationViewSet, basename=Constants.ORGANIZATION) +router.register(r"team_member", TeamMemberViewSet, basename=Constants.TEAM_MEMBER) +router.register("drop_document", DropDocumentView, basename=Constants.DROP_DOCUMENT) +router.register("save_documents", DocumentSaveView, basename=Constants.SAVE_DOCUMENTS) +router.register("theme", DatahubThemeView, basename=Constants.THEME) +router.register(r"support", SupportViewSet, basename=Constants.SUPPORT_TICKETS) +router.register(r"", DatahubDashboard, basename="") +router.register(r"dataset/v2", DatasetV2ViewSet, basename=Constants.DATASET_V2_URL) +router.register(r"new_dataset_v2", DatasetV2View, basename=Constants.DATASETS_V2_URL) +router.register(r"dataset_files", DatasetFileV2View, basename=Constants.DATASET_FILES) +router.register(r"dataset_ops", DatasetV2ViewSetOps, basename="") +router.register(r"standardise", StandardisationTemplateView, basename=Constants.STANDARDISE) +router.register(r"newdashboard", DatahubNewDashboard, basename=Constants.NEW_DASHBOARD) +router.register(r"resource_management", ResourceManagementViewSet, basename=Constants.RESOURCE_MANAGEMENT) +router.register(r"resource_file", ResourceFileManagementViewSet, basename=Constants.RESOURCE_FILE_MANAGEMENT) +router.register(r'categories', CategoryViewSet, basename=Constants.CATEGORY) +router.register(r'subcategories', SubCategoryViewSet, basename=Constants.SUBCATEGORY) +router.register(r'embeddings', EmbeddingsViewSet, basename='embeddings') urlpatterns = [ path("", include(router.urls)), + path('policy/', PolicyListAPIView.as_view(), name='policy-list'), + path('policy//', PolicyDetailAPIView.as_view(), name='policy-detail'), + path('usage_policies/', UsagePolicyListCreateView.as_view(), name='usage-policy-list-create'), + path('usage_policies//', UsagePolicyRetrieveUpdateDestroyView.as_view(), name='usage-policy-retrieve-update-destroy'), + path('resource_usage_policies/', ResourceUsagePolicyListCreateView.as_view(), name='resource_usage-policy-list-create'), + path('resource_usage_policies//', ResourceUsagePolicyRetrieveUpdateDestroyView.as_view(), name='resource_usage-policy-retrieve-update-destroy'), + path('messages//', MessagesViewSet.as_view(), name='messages-retrieve-update-destroy'), + path('messages/', MessagesCreateViewSet.as_view(), name='messages_create'), ] # from django.urls import path diff --git a/participant/views.py b/participant/views.py index 16f5583e..96c07781 100644 --- a/participant/views.py +++ b/participant/views.py @@ -5,14 +5,18 @@ import operator import os import re +import shutil import subprocess +import threading import time from bdb import set_trace +from calendar import c from contextlib import closing from functools import reduce from sre_compile import isstring from struct import unpack from timeit import Timer +from urllib.parse import unquote import mysql.connector import pandas as pd @@ -20,53 +24,68 @@ import requests import xlwt from django.conf import settings -from django.db.models import Q -from django.db.models.functions import Lower +from django.core.cache import cache +from django.core.files.base import ContentFile +from django.db import transaction +from django.db.models import Q, F, Count, Sum, Func, CharField, Value, Subquery +from django.db.models.functions import Lower, Concat from django.http import HttpResponse, JsonResponse from django.shortcuts import render from psycopg2 import errorcodes -from rest_framework import pagination, serializers, status +from python_http_client import exceptions +from rest_framework import pagination, serializers, status, generics, viewsets from rest_framework.decorators import action, api_view, permission_classes from rest_framework.exceptions import ValidationError from rest_framework.generics import get_object_or_404 -from rest_framework.parsers import JSONParser -from rest_framework.permissions import IsAuthenticated +from rest_framework.parsers import JSONParser, MultiPartParser +from rest_framework.permissions import IsAuthenticated, AllowAny from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet, ViewSet from uritemplate import partial from accounts.models import User +from accounts.serializers import UserCreateSerializer from core.constants import Constants, NumericalConstants +from core.serializer_validation import OrganizationSerializerValidator, UserCreateSerializerValidator from core.utils import ( CustomPagination, - DefaultPagination, Utils, csv_and_xlsx_file_validatation, date_formater, one_day_date_formater, read_contents_from_csv_or_xlsx_file, - timer, + timer, generate_hash_key_for_dashboard, generate_api_key, ) -from datahub.models import ( - Datasets, +from participant.models import ( DatasetV2, DatasetV2File, Organization, UserOrganizationMap, ) -from datahub.serializers import DatasetFileV2NewSerializer + +from datasets.models import Datasets +from participant.serializers import DatasetFileV2NewSerializer + from participant.internal_services.support_ticket_internal_services import ( SupportTicketInternalServices, ) + + +# TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT from participant.models import ( - Connectors, - ConnectorsMap, + # Connectors, + # ConnectorsMap, Department, Project, Resolution, SupportTicket, - SupportTicketV2, + SupportTicketV2, DatahubDocuments, DatasetSubCategoryMap, SubCategory, StandardisationTemplate, Policy, UsagePolicy, + Resource, ResourceSubCategoryMap, ResourceUsagePolicy, ResourceFile, LangchainPgCollection, Category, + LangchainPgEmbedding, Messages, ) + + +# TODO - REMOVED IMOPORT TO CONNECTOR MODEL TO AVOID CIRCULAR IMPORT from participant.serializers import ( ConnectorListSerializer, ConnectorsConsumerRelationSerializer, @@ -76,7 +95,7 @@ ConnectorsMapSerializer, ConnectorsProviderRelationSerializer, ConnectorsRetriveSerializer, - ConnectorsSerializer, + # ConnectorsSerializer, ConnectorsSerializerForEmail, CreateSupportTicketResolutionsSerializer, CreateSupportTicketV2Serializer, @@ -96,18 +115,38 @@ SupportTicketResolutionsSerializerMinimised, SupportTicketV2Serializer, TicketSupportSerializer, - UpdateSupportTicketV2Serializer, + UpdateSupportTicketV2Serializer, TeamMemberListSerializer, TeamMemberCreateSerializer, TeamMemberDetailsSerializer, + TeamMemberUpdateSerializer, OrganizationSerializerExhaustive, UserOrganizationMapSerializer, ParticipantSerializer, + ParticipantCostewardSerializer, DropDocumentSerializer, PolicyDocumentSerializer, DatahubThemeSerializer, + DatasetV2Serializer, DatasetV2Validation, DatasetV2TempFileSerializer, UsagePolicyDetailSerializer, + DatahubDatasetsV2Serializer, StandardisationTemplateViewSerializer, PolicySerializer, DatasetV2NewListSerializer, + UsagePolicySerializer, UsageUpdatePolicySerializer, APIBuilderSerializer, DatasetV2ListNewSerializer, + ResourceSerializer, ResourceListSerializer, ResourceFileSerializer, CategorySerializer, MessagesRetriveSerializer, + ResourceUsagePolicySerializer, ResourceAPIBuilderSerializer, MessagesChunksRetriveSerializer, MessagesSerializer, + LangChainEmbeddingsSerializer, SubCategorySerializer, DatasetV2DetailNewSerializer, + StandardisationTemplateUpdateSerializer, RecentDatasetListSerializer, RecentSupportTicketSerializer, + DatasetUpdateSerializer, DatahubDatasetsSerializer, ) -from utils import file_operations as file_ops +from utils import file_operations as file_ops, file_operations, custom_exceptions from utils import string_functions +from utils.authentication_services import authenticate_user from utils.authorization_services import support_ticket_role_authorization +from utils.embeddings_creation import VectorDBBuilder # from utils.connector_utils import run_containers, stop_containers -from utils.file_operations import check_file_name_length +from utils.file_operations import check_file_name_length, filter_dataframe_for_dashboard_counties, \ + generate_omfp_dashboard, generate_fsp_dashboard, generate_knfd_dashboard from utils.jwt_services import http_request_mutation +from utils.youtube_helper import get_youtube_url LOGGER = logging.getLogger(__name__) +class DefaultPagination(pagination.PageNumberPagination): + """ + Configure Pagination + """ + + page_size = 5 class ParticipantSupportViewSet(GenericViewSet): """ @@ -682,8 +721,8 @@ class ParticipantConnectorsViewSet(GenericViewSet): """ parser_class = JSONParser - serializer_class = ConnectorsSerializer - queryset = Connectors + # serializer_class = ConnectorsSerializer + # queryset = Connectors pagination_class = CustomPagination def perform_create(self, serializer): @@ -1083,8 +1122,8 @@ class ParticipantConnectorsMapViewSet(GenericViewSet): """ parser_class = JSONParser - serializer_class = ConnectorsMapSerializer - queryset = ConnectorsMap + # serializer_class = ConnectorsMapSerializer + # queryset = ConnectorsMap pagination_class = CustomPagination def perform_create(self, serializer): @@ -2333,3 +2372,3608 @@ def destroy(self, request, pk=None): except Exception as e: LOGGER.error(e, exc_info=True) return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + + +# LOGGER = logging.getLogger(__name__) +con = None + + + + + +class TeamMemberViewSet(GenericViewSet): + """Viewset for Product model""" + + serializer_class = TeamMemberListSerializer + queryset = User.objects.all() + pagination_class = CustomPagination + + def create(self, request, *args, **kwargs): + """POST method: create action to save an object by sending a POST request""" + try: + serializer = TeamMemberCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def list(self, request, *args, **kwargs): + """GET method: query all the list of objects from the Product model""" + # queryset = self.filter_queryset(self.get_queryset()) + queryset = User.objects.filter(Q(status=True) & (Q(role__id=2) | Q(role__id=5))) + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + def retrieve(self, request, pk): + """GET method: retrieve an object or instance of the Product model""" + team_member = self.get_object() + serializer = TeamMemberDetailsSerializer(team_member) + # serializer.is_valid(raise_exception=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + def update(self, request, *args, **kwargs): + """PUT method: update or send a PUT request on an object of the Product model""" + try: + instance = self.get_object() + # request.data["role"] = UserRole.objects.get(role_name=request.data["role"]).id + serializer = TeamMemberUpdateSerializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def destroy(self, request, pk): + """DELETE method: delete an object""" + team_member = self.get_object() + team_member.status = False + # team_member.delete() + team_member.save() + return Response(status=status.HTTP_204_NO_CONTENT) + + +class OrganizationViewSet(GenericViewSet): + """ + Organisation Viewset. + """ + + serializer_class = OrganizationSerializerExhaustive + queryset = Organization.objects.all() + pagination_class = CustomPagination + parser_class = MultiPartParser + + def perform_create(self, serializer): + """ + This function performs the create operation of requested serializer. + Args: + serializer (_type_): serializer class object. + + Returns: + _type_: Returns the saved details. + """ + return serializer.save() + + def create(self, request, *args, **kwargs): + """POST method: create action to save an organization object using User ID (IMPORTANT: Using USER ID instead of Organization ID)""" + try: + user_obj = User.objects.get(id=request.data.get(Constants.USER_ID)) + user_org_queryset = UserOrganizationMap.objects.filter(user_id=request.data.get(Constants.USER_ID)).first() + if user_org_queryset: + return Response( + {"message": ["User is already associated with an organization"]}, + status=status.HTTP_400_BAD_REQUEST, + ) + else: + with transaction.atomic(): + # create organization and userorganizationmap object + print("Creating org & user_org_map") + OrganizationSerializerValidator.validate_website(request.data) + org_serializer = OrganizationSerializerExhaustive(data=request.data, partial=True) + org_serializer.is_valid(raise_exception=True) + org_queryset = self.perform_create(org_serializer) + + user_org_serializer = UserOrganizationMapSerializer( + data={ + Constants.USER: user_obj.id, + Constants.ORGANIZATION: org_queryset.id, + } # type: ignore + ) + user_org_serializer.is_valid(raise_exception=True) + self.perform_create(user_org_serializer) + data = { + "user_map": user_org_serializer.data.get("id"), + "org_id": org_queryset.id, + "organization": org_serializer.data, + } + return Response(data, status=status.HTTP_201_CREATED) + + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def list(self, request, *args, **kwargs): + """GET method: query the list of Organization objects""" + try: + user_org_queryset = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter(organization__status=True) + .all() + ) + page = self.paginate_queryset(user_org_queryset) + user_organization_serializer = ParticipantSerializer(page, many=True) + return self.get_paginated_response(user_organization_serializer.data) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def retrieve(self, request, pk): + """GET method: retrieve an object of Organization using User ID of the User (IMPORTANT: Using USER ID instead of Organization ID)""" + try: + user_obj = User.objects.get(id=pk, status=True) + user_org_queryset = UserOrganizationMap.objects.prefetch_related( + Constants.USER, Constants.ORGANIZATION + ).filter(user=pk) + + if not user_org_queryset: + data = {Constants.USER: {"id": user_obj.id}, Constants.ORGANIZATION: "null"} + return Response(data, status=status.HTTP_200_OK) + + org_obj = Organization.objects.get(id=user_org_queryset.first().organization_id) + user_org_serializer = OrganizationSerializerExhaustive(org_obj) + data = { + Constants.USER: {"id": user_obj.id}, + Constants.ORGANIZATION: user_org_serializer.data, + } + return Response(data, status=status.HTTP_200_OK) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def update(self, request, pk): + """PUT method: update or PUT request for Organization using User ID of the User (IMPORTANT: Using USER ID instead of Organization ID)""" + user_obj = User.objects.get(id=pk, status=True) + user_org_queryset = ( + UserOrganizationMap.objects.prefetch_related(Constants.USER, Constants.ORGANIZATION).filter(user=pk).all() + ) + + if not user_org_queryset: + return Response({}, status=status.HTTP_404_NOT_FOUND) # 310-360 not covered 4 + OrganizationSerializerValidator.validate_website(request.data) + organization_serializer = OrganizationSerializerExhaustive( + Organization.objects.get(id=user_org_queryset.first().organization_id), + data=request.data, + partial=True, + ) + try: + organization_serializer.is_valid(raise_exception=True) + self.perform_create(organization_serializer) + data = { + Constants.USER: {"id": pk}, + Constants.ORGANIZATION: organization_serializer.data, + "user_map": user_org_queryset.first().id, + "org_id": user_org_queryset.first().organization_id, + } + return Response( + data, + status=status.HTTP_201_CREATED, + ) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def destroy(self, request, pk): + """DELETE method: delete an object""" + try: + user_obj = User.objects.get(id=pk, status=True) + user_org_queryset = UserOrganizationMap.objects.select_related(Constants.ORGANIZATION).get(user_id=pk) + org_queryset = Organization.objects.get(id=user_org_queryset.organization_id) + org_queryset.status = False + self.perform_create(org_queryset) + return Response(status=status.HTTP_204_NO_CONTENT) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class ParticipantViewSet(GenericViewSet): + """ + This class handles the participant CRUD operations. + """ + + parser_class = JSONParser + serializer_class = UserCreateSerializer + queryset = User.objects.all() + pagination_class = CustomPagination + + def perform_create(self, serializer): + """ + This function performs the create operation of requested serializer. + Args: + serializer (_type_): serializer class object. + + Returns: + _type_: Returns the saved details. + """ + return serializer.save() + + @authenticate_user(model=Organization) + @transaction.atomic + def create(self, request, *args, **kwargs): + """POST method: create action to save an object by sending a POST request""" + OrganizationSerializerValidator.validate_website(request.data) + org_serializer = UserCreateSerializer(data=request.data, partial=True) + org_serializer.is_valid(raise_exception=True) + org_queryset = self.perform_create(org_serializer) + org_id = org_queryset.id + UserCreateSerializerValidator.validate_phone_number_format(request.data) + user_serializer = UserCreateSerializer(data=request.data) + user_serializer.is_valid(raise_exception=True) + user_saved = self.perform_create(user_serializer) + user_org_serializer = UserOrganizationMapSerializer( + data={ + Constants.USER: user_saved.id, + Constants.ORGANIZATION: org_id, + } # type: ignore + ) + user_org_serializer.is_valid(raise_exception=True) + self.perform_create(user_org_serializer) + try: + if user_saved.on_boarded_by: + # datahub_admin = User.objects.filter(id=user_saved.on_boarded_by).first() + admin_full_name = string_functions.get_full_name( + user_saved.on_boarded_by.first_name, + user_saved.on_boarded_by.last_name, + ) + else: + datahub_admin = User.objects.filter(role_id=1).first() + admin_full_name = string_functions.get_full_name(datahub_admin.first_name, datahub_admin.last_name) + participant_full_name = string_functions.get_full_name( + request.data.get("first_name"), request.data.get("last_name") + ) + data = { + Constants.datahub_name: os.environ.get(Constants.DATAHUB_NAME, Constants.datahub_name), + "as_user": "Co-Steward" if user_saved.role == 6 else "Participant", + "participant_admin_name": participant_full_name, + "participant_organization_name": request.data.get("name"), + "datahub_admin": admin_full_name, + Constants.datahub_site: os.environ.get(Constants.DATAHUB_SITE, Constants.datahub_site), + } + + email_render = render(request, Constants.WHEN_DATAHUB_ADMIN_ADDS_PARTICIPANT, data) + mail_body = email_render.content.decode("utf-8") + Utils().send_email( + to_email=request.data.get("email"), + content=mail_body, + subject=Constants.PARTICIPANT_ORG_ADDITION_SUBJECT + + os.environ.get(Constants.DATAHUB_NAME, Constants.datahub_name), + ) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + return Response(user_org_serializer.data, status=status.HTTP_201_CREATED) + + def list(self, request, *args, **kwargs): + """GET method: query all the list of objects from the Product model""" + on_boarded_by = request.GET.get("on_boarded_by", None) + co_steward = request.GET.get("co_steward", False) + approval_status = request.GET.get(Constants.APPROVAL_STATUS, True) + name = request.GET.get(Constants.NAME, "") + filter = {Constants.ORGANIZATION_NAME_ICONTAINS: name} if name else {} + if on_boarded_by: + roles = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter( + user__status=True, + user__on_boarded_by=on_boarded_by, + user__role=3, + user__approval_status=approval_status, + **filter, + ) + .order_by("-user__updated_at") + .all() + ) + elif co_steward: + roles = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter(user__status=True, user__role=6, **filter) + .order_by("-user__updated_at") + .all() + ) + page = self.paginate_queryset(roles) + participant_serializer = ParticipantCostewardSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + else: + roles = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter( + user__status=True, + user__role=3, + user__on_boarded_by=None, + user__approval_status=approval_status, + **filter, + ) + .order_by("-user__updated_at") + .all() + ) + + page = self.paginate_queryset(roles) + participant_serializer = ParticipantSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + + def retrieve(self, request, pk): + """GET method: retrieve an object or instance of the Product model""" + roles = ( + UserOrganizationMap.objects.prefetch_related(Constants.USER, Constants.ORGANIZATION) + .filter(user__status=True, user=pk) + .first() + ) + + participant_serializer = ParticipantSerializer(roles, many=False) + if participant_serializer.data: + return Response(participant_serializer.data, status=status.HTTP_200_OK) + return Response([], status=status.HTTP_200_OK) + + @authenticate_user(model=Organization) + @transaction.atomic + def update(self, request, *args, **kwargs): + """PUT method: update or send a PUT request on an object of the Product model""" + try: + participant = self.get_object() + user_serializer = self.get_serializer(participant, data=request.data, partial=True) + user_serializer.is_valid(raise_exception=True) + organization = Organization.objects.get(id=request.data.get(Constants.ID)) + OrganizationSerializerValidator.validate_website(request.data) + organization_serializer = OrganizationSerializerExhaustive(organization, data=request.data, partial=True) + organization_serializer.is_valid(raise_exception=True) + user_data = self.perform_create(user_serializer) + self.perform_create(organization_serializer) + + if user_data.on_boarded_by: + admin_full_name = string_functions.get_full_name(user_data.first_name, user_data.last_name) + else: + datahub_admin = User.objects.filter(role_id=1).first() + admin_full_name = string_functions.get_full_name(datahub_admin.first_name, datahub_admin.last_name) + participant_full_name = string_functions.get_full_name(participant.first_name, participant.last_name) + + data = { + Constants.datahub_name: os.environ.get(Constants.DATAHUB_NAME, Constants.datahub_name), + "participant_admin_name": participant_full_name, + "participant_organization_name": organization.name, + "datahub_admin": admin_full_name, + Constants.datahub_site: os.environ.get(Constants.DATAHUB_SITE, Constants.datahub_site), + } + + # update data & trigger_email + email_render = render(request, Constants.DATAHUB_ADMIN_UPDATES_PARTICIPANT_ORGANIZATION, data) + mail_body = email_render.content.decode("utf-8") + Utils().send_email( + to_email=participant.email, + content=mail_body, + subject=Constants.PARTICIPANT_ORG_UPDATION_SUBJECT + + os.environ.get(Constants.DATAHUB_NAME, Constants.datahub_name), + ) + + data = { + Constants.USER: user_serializer.data, + Constants.ORGANIZATION: organization_serializer.data, + } + return Response(data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @authenticate_user(model=Organization) + def destroy(self, request, pk): + """DELETE method: delete an object""" + participant = self.get_object() + user_organization = ( + UserOrganizationMap.objects.select_related(Constants.ORGANIZATION).filter(user_id=pk).first() + ) + organization = Organization.objects.get(id=user_organization.organization_id) + if participant.status: + participant.status = False + try: + if participant.on_boarded_by: + datahub_admin = participant.on_boarded_by + else: + datahub_admin = User.objects.filter(role_id=1).first() + admin_full_name = string_functions.get_full_name(datahub_admin.first_name, datahub_admin.last_name) + participant_full_name = string_functions.get_full_name(participant.first_name, participant.last_name) + + data = { + Constants.datahub_name: os.environ.get(Constants.DATAHUB_NAME, Constants.datahub_name), + "participant_admin_name": participant_full_name, + "participant_organization_name": organization.name, + "datahub_admin": admin_full_name, + Constants.datahub_site: os.environ.get(Constants.DATAHUB_SITE, Constants.datahub_site), + } + + # delete data & trigger_email + self.perform_create(participant) + email_render = render( + request, + Constants.DATAHUB_ADMIN_DELETES_PARTICIPANT_ORGANIZATION, + data, + ) + mail_body = email_render.content.decode("utf-8") + Utils().send_email( + to_email=participant.email, + content=mail_body, + subject=Constants.PARTICIPANT_ORG_DELETION_SUBJECT + + os.environ.get(Constants.DATAHUB_NAME, Constants.datahub_name), + ) + + # Set the on_boarded_by_id to null if co_steward is deleted + User.objects.filter(on_boarded_by=pk).update(on_boarded_by=None) + + return Response( + {"message": ["Participant deleted"]}, + status=status.HTTP_204_NO_CONTENT, + ) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response({"message": ["Internal server error"]}, status=500) + + elif participant.status is False: + return Response( + {"message": ["participant/co-steward already deleted"]}, + status=status.HTTP_204_NO_CONTENT, + ) + + return Response({"message": ["Internal server error"]}, status=500) + + @action(detail=False, methods=["post"], permission_classes=[AllowAny]) + def get_list_co_steward(self, request, *args, **kwargs): + try: + users = ( + User.objects.filter(role__id=6, status=True) + .values("id", "userorganizationmap__organization__name") + .distinct("userorganizationmap__organization__name") + ) + + data = [ + { + "user": user["id"], + "organization_name": user["userorganizationmap__organization__name"], + } + for user in users + ] + return Response(data, status=200) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response({"message": str(e)}, status=500) + + +class MailInvitationViewSet(GenericViewSet): + """ + This class handles the mail invitation API views. + """ + + def create(self, request, *args, **kwargs): + """ + This will send the mail to the requested user with content. + Args: + request (_type_): Api request object. + + Returns: + _type_: Retuns the sucess response with message and status code. + """ + try: + email_list = request.data.get("to_email") + emails_found, emails_not_found = ([] for i in range(2)) + # for email in email_list: + # if User.objects.filter(email=email): + # emails_found.append(email) + # else: + # emails_not_found.append(email) + user = User.objects.filter(role_id=1).first() + full_name = user.first_name + " " + str(user.last_name) if user.last_name else user.first_name + data = { + "datahub_name": os.environ.get("DATAHUB_NAME", "datahub_name"), + "participant_admin_name": full_name, + "datahub_site": os.environ.get("DATAHUB_SITE", "datahub_site"), + } + # render email from query_email template + for email in email_list: + try: + email_render = render(request, "datahub_admin_invites_participants.html", data) + mail_body = email_render.content.decode("utf-8") + Utils().send_email( + to_email=[email], + content=mail_body, + subject=os.environ.get("DATAHUB_NAME", "datahub_name") + + Constants.PARTICIPANT_INVITATION_SUBJECT, + ) + except Exception as e: + emails_not_found.append() + failed = f"No able to send emails to this emails: {emails_not_found}" + LOGGER.warning(failed) + return Response( + { + "message": f"Email successfully sent to {emails_found}", + "failed": failed, + }, + status=status.HTTP_200_OK, + ) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response( + {"Error": f"Failed to send email"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) # type: ignore + + +class DropDocumentView(GenericViewSet): + """View for uploading organization document files""" + + parser_class = MultiPartParser + serializer_class = DropDocumentSerializer + + def create(self, request, *args, **kwargs): + """Saves the document files in temp location before saving""" + serializer = self.get_serializer(data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + + try: + # get file, file name & type from the form-data + key = list(request.data.keys())[0] + file = serializer.validated_data[key] + file_type = str(file).split(".")[-1] + file_name = str(key) + "." + file_type + file_operations.remove_files(file_name, settings.TEMP_FILE_PATH) + file_operations.file_save(file, file_name, settings.TEMP_FILE_PATH) + return Response( + {key: [f"{file_name} uploading in progress ..."]}, + status=status.HTTP_201_CREATED, + ) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["delete"]) + def delete(self, request): + """remove the dropped documents""" + try: + key = list(request.data.keys())[0] + file_operations.remove_files(key, settings.TEMP_FILE_PATH) + return Response({}, status=status.HTTP_204_NO_CONTENT) + + except Exception as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + +class DocumentSaveView(GenericViewSet): + """View for uploading all the datahub documents and content""" + + serializer_class = PolicyDocumentSerializer + queryset = DatahubDocuments.objects.all() + + @action(detail=False, methods=["get"]) + def get(self, request): + """GET method: retrieve an object or instance of the Product model""" + try: + file_paths = file_operations.file_path(settings.DOCUMENTS_URL) + datahub_obj = DatahubDocuments.objects.last() + content = { + Constants.GOVERNING_LAW: datahub_obj.governing_law if datahub_obj else None, + Constants.PRIVACY_POLICY: datahub_obj.privacy_policy if datahub_obj else None, + Constants.TOS: datahub_obj.tos if datahub_obj else None, + Constants.LIMITATIONS_OF_LIABILITIES: datahub_obj.limitations_of_liabilities if datahub_obj else None, + Constants.WARRANTY: datahub_obj.warranty if datahub_obj else None, + } + + documents = { + Constants.GOVERNING_LAW: file_paths.get("governing_law"), + Constants.PRIVACY_POLICY: file_paths.get("privacy_policy"), + Constants.TOS: file_paths.get("tos"), + Constants.LIMITATIONS_OF_LIABILITIES: file_paths.get("limitations_of_liabilities"), + Constants.WARRANTY: file_paths.get("warranty"), + } + if not datahub_obj and not file_paths: + data = {"content": content, "documents": documents} + return Response(data, status=status.HTTP_200_OK) + elif not datahub_obj: + data = {"content": content, "documents": documents} + return Response(data, status=status.HTTP_200_OK) + elif datahub_obj and not file_paths: + data = {"content": content, "documents": documents} + return Response(data, status=status.HTTP_200_OK) + elif datahub_obj and file_paths: + data = {"content": content, "documents": documents} + return Response(data, status=status.HTTP_200_OK) + + except Exception as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_404_NOT_FOUND) + + def create(self, request, *args, **kwargs): + try: + serializer = self.get_serializer(data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + + with transaction.atomic(): + serializer.save() + # save the document files + file_operations.create_directory(settings.DOCUMENTS_ROOT, []) + file_operations.files_move(settings.TEMP_FILE_PATH, settings.DOCUMENTS_ROOT) + return Response( + {"message": "Documents and content saved!"}, + status=status.HTTP_201_CREATED, + ) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["get"]) + def put(self, request, *args, **kwargs): + """Saves the document content and files""" + try: + # instance = self.get_object() + datahub_obj = DatahubDocuments.objects.last() + serializer = self.get_serializer(datahub_obj, data=request.data) + serializer.is_valid(raise_exception=True) + + with transaction.atomic(): + serializer.save() + file_operations.create_directory(settings.DOCUMENTS_ROOT, []) + file_operations.files_move(settings.TEMP_FILE_PATH, settings.DOCUMENTS_ROOT) + return Response( + {"message": "Documents and content updated!"}, + status=status.HTTP_201_CREATED, + ) + except Exception as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + +class DatahubThemeView(GenericViewSet): + """View for modifying datahub branding""" + + parser_class = MultiPartParser + serializer_class = DatahubThemeSerializer + + def create(self, request, *args, **kwargs): + """generates the override css for datahub""" + # user = User.objects.filter(email=request.data.get("email", "")) + # user = user.first() + data = {} + + try: + banner = request.data.get("banner", "null") + banner = None if banner == "null" else banner + button_color = request.data.get("button_color", "null") + button_color = None if button_color == "null" else button_color + if not banner and not button_color: + data = {"banner": "null", "button_color": "null"} + elif banner and not button_color: + file_name = file_operations.file_rename(str(banner), "banner") + shutil.rmtree(settings.THEME_ROOT) + os.mkdir(settings.THEME_ROOT) + os.makedirs(settings.CSS_ROOT) + file_operations.file_save(banner, file_name, settings.THEME_ROOT) + data = {"banner": file_name, "button_color": "null"} + + elif not banner and button_color: + css = ".btn { background-color: " + button_color + "; }" + file_operations.remove_files(file_name, settings.THEME_ROOT) + file_operations.file_save( + ContentFile(css), + settings.CSS_FILE_NAME, + settings.CSS_ROOT, + ) + data = {"banner": "null", "button_color": settings.CSS_FILE_NAME} + + elif banner and button_color: + shutil.rmtree(settings.THEME_ROOT) + os.mkdir(settings.THEME_ROOT) + os.makedirs(settings.CSS_ROOT) + file_name = file_operations.file_rename(str(banner), "banner") + file_operations.remove_files(file_name, settings.THEME_ROOT) + file_operations.file_save(banner, file_name, settings.THEME_ROOT) + + css = ".btn { background-color: " + button_color + "; }" + file_operations.remove_files(file_name, settings.THEME_ROOT) + file_operations.file_save( + ContentFile(css), + settings.CSS_FILE_NAME, + settings.CSS_ROOT, + ) + data = {"banner": file_name, "button_color": settings.CSS_FILE_NAME} + + # set datahub admin user status to True + # user.on_boarded = True + # user.save() + return Response(data, status=status.HTTP_201_CREATED) + + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["get"]) + def get(self, request): + """retrieves Datahub Theme attributes""" + file_paths = file_operations.file_path(settings.THEME_URL) + # css_path = file_operations.file_path(settings.CSS_ROOT) + css_path = settings.CSS_ROOT + settings.CSS_FILE_NAME + data = {} + + try: + css_attribute = file_operations.get_css_attributes(css_path, "background-color") + + if not css_path and not file_paths: + data = {"banner": "null", "css": "null"} + elif not css_path: + data = {"banner": file_paths, "css": "null"} + elif css_path and not file_paths: + data = {"banner": "null", "css": {"btnBackground": css_attribute}} + elif css_path and file_paths: + data = {"banner": file_paths, "css": {"btnBackground": css_attribute}} + + return Response(data, status=status.HTTP_200_OK) + + except Exception as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False) + def put(self, request, *args, **kwargs): + data = {} + try: + banner = request.data.get("banner", "null") + banner = None if banner == "null" else banner + button_color = request.data.get("button_color", "null") + button_color = None if button_color == "null" else button_color + + if banner is None and button_color is None: + data = {"banner": "null", "button_color": "null"} + + elif banner and button_color is None: + shutil.rmtree(settings.THEME_ROOT) + os.mkdir(settings.THEME_ROOT) + os.makedirs(settings.CSS_ROOT) + file_name = file_operations.file_rename(str(banner), "banner") + # file_operations.remove_files(file_name, settings.THEME_ROOT) + file_operations.file_save(banner, file_name, settings.THEME_ROOT) + data = {"banner": file_name, "button_color": "null"} + + elif not banner and button_color: + css = ".btn { background-color: " + button_color + "; }" + file_operations.remove_files(settings.CSS_FILE_NAME, settings.CSS_ROOT) + file_operations.file_save( + ContentFile(css), + settings.CSS_FILE_NAME, + settings.CSS_ROOT, + ) + data = {"banner": "null", "button_color": settings.CSS_FILE_NAME} + + elif banner and button_color: + shutil.rmtree(settings.THEME_ROOT) + os.mkdir(settings.THEME_ROOT) + os.makedirs(settings.CSS_ROOT) + file_name = file_operations.file_rename(str(banner), "banner") + # file_operations.remove_files(file_name, settings.THEME_ROOT) + file_operations.file_save(banner, file_name, settings.THEME_ROOT) + + css = ".btn { background-color: " + button_color + "; }" + file_operations.remove_files(settings.CSS_FILE_NAME, settings.CSS_ROOT) + file_operations.file_save( + ContentFile(css), + settings.CSS_FILE_NAME, + settings.CSS_ROOT, + ) + data = {"banner": file_name, "button_color": settings.CSS_FILE_NAME} + + return Response(data, status=status.HTTP_201_CREATED) + + except exceptions as error: + LOGGER.error(error, exc_info=True) + + return Response({}, status=status.HTTP_400_BAD_REQUEST) + + +class SupportViewSet(GenericViewSet): + """ + This class handles the participant support tickets CRUD operations. + """ + + parser_class = JSONParser + serializer_class = TicketSupportSerializer + queryset = SupportTicket + pagination_class = CustomPagination + + def perform_create(self, serializer): + """ + This function performs the create operation of requested serializer. + Args: + serializer (_type_): serializer class object. + + Returns: + _type_: Returns the saved details. + """ + return serializer.save() + + def create(self, request, *args, **kwargs): + """POST method: create action to save an object by sending a POST request""" + try: + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=False, methods=["post"]) + def filters_tickets(self, request, *args, **kwargs): + """This function get the filter args in body. based on the filter args orm filters the data.""" + try: + data = ( + SupportTicket.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, **request.data) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + except django.core.exceptions.FieldError as error: # type: ignore + LOGGER.error(f"Error while filtering the ticketd ERROR: {error}") + return Response(f"Invalid filter fields: {list(request.data.keys())}", status=400) + + page = self.paginate_queryset(data) + participant_serializer = ParticipantSupportTicketSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + + def update(self, request, *args, **kwargs): + """PUT method: update or send a PUT request on an object of the Product model""" + try: + instance = self.get_object() + serializer = self.get_serializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def list(self, request, *args, **kwargs): + """GET method: query all the list of objects from the Product model""" + data = ( + SupportTicket.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, **request.GET) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + page = self.paginate_queryset(data) + participant_serializer = ParticipantSupportTicketSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + + def retrieve(self, request, pk): + """GET method: retrieve an object or instance of the Product model""" + data = ( + SupportTicket.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, id=pk) + .all() + ) + participant_serializer = ParticipantSupportTicketSerializer(data, many=True) + if participant_serializer.data: + return Response(participant_serializer.data[0], status=status.HTTP_200_OK) + return Response([], status=status.HTTP_200_OK) + + +class DatahubDatasetsViewSet(GenericViewSet): + """ + This class handles the participant Datsets CRUD operations. + """ + + parser_class = JSONParser + serializer_class = DatasetSerializer + queryset = Datasets + pagination_class = CustomPagination + + def perform_create(self, serializer): + """ + This function performs the create operation of requested serializer. + Args: + serializer (_type_): serializer class object. + + Returns: + _type_: Returns the saved details. + """ + return serializer.save() + + def trigger_email(self, request, template, to_email, subject, first_name, last_name, dataset_name): + # trigger email to the participant as they are being added + try: + datahub_admin = User.objects.filter(role_id=1).first() + admin_full_name = string_functions.get_full_name(datahub_admin.first_name, datahub_admin.last_name) + participant_full_name = string_functions.get_full_name(first_name, last_name) + + data = { + "datahub_name": os.environ.get("DATAHUB_NAME", "datahub_name"), + "participant_admin_name": participant_full_name, + "datahub_admin": admin_full_name, + "dataset_name": dataset_name, + "datahub_site": os.environ.get("DATAHUB_SITE", "datahub_site"), + } + + email_render = render(request, template, data) + mail_body = email_render.content.decode("utf-8") + Utils().send_email( + to_email=to_email, + content=mail_body, + subject=subject + os.environ.get("DATAHUB_NAME", "datahub_name"), + ) + + except Exception as error: + LOGGER.error(error, exc_info=True) + + def create(self, request, *args, **kwargs): + """POST method: create action to save an object by sending a POST request""" + setattr(request.data, "_mutable", True) + data = request.data + + if not data.get("is_public"): + if not csv_and_xlsx_file_validatation(request.data.get(Constants.SAMPLE_DATASET)): + return Response( + { + Constants.SAMPLE_DATASET: [ + "Invalid Sample dataset file (or) Atleast 5 rows should be available. please upload valid file" + ] + }, + 400, + ) + try: + data[Constants.APPROVAL_STATUS] = Constants.APPROVED + serializer = self.get_serializer(data=data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @http_request_mutation + def list(self, request, *args, **kwargs): + """GET method: query all the list of objects from the Product model""" + try: + data = [] + user_id = request.META.get(Constants.USER_ID) + others = request.data.get(Constants.OTHERS) + filters = {Constants.USER_MAP_USER: user_id} if user_id and not others else {} + exclude = {Constants.USER_MAP_USER: user_id} if others else {} + if exclude or filters: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, status=True, **filters) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + page = self.paginate_queryset(data) + participant_serializer = DatahubDatasetsSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + def retrieve(self, request, pk): + """GET method: retrieve an object or instance of the Product model""" + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, status=True, id=pk) + .all() + ) + participant_serializer = DatahubDatasetsSerializer(data, many=True) + if participant_serializer.data: + data = participant_serializer.data[0] + if not data.get("is_public"): + data[Constants.CONTENT] = read_contents_from_csv_or_xlsx_file(data.get(Constants.SAMPLE_DATASET)) + return Response(data, status=status.HTTP_200_OK) + return Response({}, status=status.HTTP_200_OK) + + def update(self, request, *args, **kwargs): + """PUT method: update or send a PUT request on an object of the Product model""" + setattr(request.data, "_mutable", True) + data = request.data + data = {key: value for key, value in data.items() if value != "null"} + if not data.get("is_public"): + if data.get(Constants.SAMPLE_DATASET): + if not csv_and_xlsx_file_validatation(data.get(Constants.SAMPLE_DATASET)): + return Response( + { + Constants.SAMPLE_DATASET: [ + "Invalid Sample dataset file (or) Atleast 5 rows should be available. please upload valid file" + ] + }, + 400, + ) + category = data.get(Constants.CATEGORY) + if category: + data[Constants.CATEGORY] = json.loads(category) if isinstance(category, str) else category + instance = self.get_object() + + # trigger email to the participant + user_map_queryset = UserOrganizationMap.objects.select_related(Constants.USER).get(id=instance.user_map_id) + user_obj = user_map_queryset.user + + # reset the approval status b/c the user modified the dataset after an approval + if getattr(instance, Constants.APPROVAL_STATUS) == Constants.APPROVED and ( + user_obj.role_id == 3 or user_obj.role_id == 4 + ): + data[Constants.APPROVAL_STATUS] = Constants.AWAITING_REVIEW + + serializer = DatasetUpdateSerializer(instance, data=data, partial=True) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + + data = request.data + + if data.get(Constants.APPROVAL_STATUS) == Constants.APPROVED: + self.trigger_email( + request, + "datahub_admin_approves_dataset.html", + user_obj.email, + Constants.APPROVED_NEW_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + elif data.get(Constants.APPROVAL_STATUS) == Constants.REJECTED: + self.trigger_email( + request, + "datahub_admin_rejects_dataset.html", + user_obj.email, + Constants.REJECTED_NEW_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + elif data.get(Constants.IS_ENABLED) == str(True) or data.get(Constants.IS_ENABLED) == str("true"): + self.trigger_email( + request, + "datahub_admin_enables_dataset.html", + user_obj.email, + Constants.ENABLE_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + elif data.get(Constants.IS_ENABLED) == str(False) or data.get(Constants.IS_ENABLED) == str("false"): + self.trigger_email( + request, + "datahub_admin_disables_dataset.html", + user_obj.email, + Constants.DISABLE_DATASET_SUBJECT, + user_obj.first_name, + user_obj.last_name, + instance.name, + ) + + return Response(serializer.data, status=status.HTTP_201_CREATED) + + def destroy(self, request, pk): + """DELETE method: delete an object""" + dataset = self.get_object() + dataset.status = False + self.perform_create(dataset) + return Response(status=status.HTTP_204_NO_CONTENT) + + @action(detail=False, methods=["post"]) + def dataset_filters(self, request, *args, **kwargs): + """This function get the filter args in body. based on the filter args orm filters the data.""" + data = request.data + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + user_id = data.pop(Constants.USER_ID, "") + categories = data.pop(Constants.CATEGORY, None) + exclude, filters = {}, {} + if others: + exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + else: + filters = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + + try: + if categories is not None: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(status=True, **data, **filters) + .filter( + reduce( + operator.or_, + (Q(category__contains=cat) for cat in categories), + ) + ) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + + else: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(status=True, **data, **filters) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + return Response(f"Invalid filter fields: {list(request.data.keys())}", status=500) + + page = self.paginate_queryset(data) + participant_serializer = DatahubDatasetsSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + + @action(detail=False, methods=["post"]) + @http_request_mutation + def filters_data(self, request, *args, **kwargs): + """This function provides the filters data""" + try: + data = request.data + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + user_id = data.pop(Constants.USER_ID, "") + + #### + + org_id = request.META.pop(Constants.ORG_ID, "") + others = request.META.pop(Constants.OTHERS, "") + user_id = request.META.pop(Constants.USER_ID, "") + + exclude, filters = {}, {} + if others: + exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + filters = {Constants.APPROVAL_STATUS: Constants.APPROVED} + else: + filters = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + try: + geography = ( + Datasets.objects.values_list(Constants.GEOGRAPHY, flat=True) + .filter(status=True, **filters) + .exclude(geography="null") + .exclude(geography__isnull=True) + .exclude(geography="") + .exclude(**exclude) + .all() + .distinct() + ) + crop_detail = ( + Datasets.objects.values_list(Constants.CROP_DETAIL, flat=True) + .filter(status=True, **filters) + .exclude(crop_detail="null") + .exclude(crop_detail__isnull=True) + .exclude(crop_detail="") + .exclude(**exclude) + .all() + .distinct() + ) + if os.path.exists(Constants.CATEGORIES_FILE): + with open(Constants.CATEGORIES_FILE, "r") as json_obj: + category_detail = json.loads(json_obj.read()) + else: + category_detail = [] + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + return Response(f"Invalid filter fields: {list(request.data.keys())}", status=500) + return Response( + { + "geography": geography, + "crop_detail": crop_detail, + "category_detail": category_detail, + }, + status=200, + ) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["post"]) + @http_request_mutation + def search_datasets(self, request, *args, **kwargs): + data = request.data + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + user_id = data.pop(Constants.USER_ID, "") + + org_id = request.META.pop(Constants.ORG_ID, "") + others = request.META.pop(Constants.OTHERS, "") + user_id = request.META.pop(Constants.USER_ID, "") + + search_pattern = data.pop(Constants.SEARCH_PATTERNS, "") + exclude, filters = {}, {} + + if others: + exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + filters = {Constants.NAME_ICONTAINS: search_pattern} if search_pattern else {} + else: + filters = ( + { + Constants.USER_MAP_ORGANIZATION: org_id, + Constants.NAME_ICONTAINS: search_pattern, + } + if org_id + else {} + ) + try: + data = ( + Datasets.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .filter(user_map__user__status=True, status=True, **data, **filters) + .exclude(**exclude) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + return Response( + f"Invalid filter fields: {list(request.data.keys())}", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + page = self.paginate_queryset(data) + participant_serializer = DatahubDatasetsSerializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + + +class DatahubDashboard(GenericViewSet): + """Datahub Dashboard viewset""" + + pagination_class = CustomPagination + + @action(detail=False, methods=["get"]) + def dashboard(self, request): + """Retrieve datahub dashboard details""" + try: + # total_participants = User.objects.filter(role_id=3, status=True).count() + total_participants = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter(user__role=3, user__status=True, is_temp=False) + .count() + ) + total_datasets = ( + DatasetV2.objects.select_related("user_map", "user_map__user", "user_map__organization") + .filter(user_map__user__status=True, is_temp=False) + .count() + ) + # write a function to compute data exchange + active_connectors = Connectors.objects.filter(status=True).count() + total_data_exchange = {"total_data": 50, "unit": "Gbs"} + + datasets = Datasets.objects.filter(status=True).values_list("category", flat=True) + categories = set() + categories_dict = {} + + for data in datasets: + if data and type(data) == dict: + for element in data.keys(): + categories.add(element) + + categories_dict = {key: 0 for key in categories} + for data in datasets: + if data and type(data) == dict: + for key, value in data.items(): + if value == True: + categories_dict[key] += 1 + + open_support_tickets = SupportTicket.objects.filter(status="open").count() + closed_support_tickets = SupportTicket.objects.filter(status="closed").count() + hold_support_tickets = SupportTicket.objects.filter(status="hold").count() + + # retrieve 3 recent support tickets + recent_tickets_queryset = SupportTicket.objects.order_by("updated_at")[0:3] + recent_tickets_serializer = RecentSupportTicketSerializer(recent_tickets_queryset, many=True) + support_tickets = { + "open_requests": open_support_tickets, + "closed_requests": closed_support_tickets, + "hold_requests": hold_support_tickets, + "recent_tickets": recent_tickets_serializer.data, + } + + # retrieve 3 recent updated datasets + # datasets_queryset = Datasets.objects.order_by("updated_at")[0:3] + datasets_queryset = Datasets.objects.filter(status=True).order_by("-updated_at").all() + datasets_queryset_pages = self.paginate_queryset(datasets_queryset) # paginaged connectors list + datasets_serializer = RecentDatasetListSerializer(datasets_queryset_pages, many=True) + + data = { + "total_participants": total_participants, + "total_datasets": total_datasets, + "active_connectors": active_connectors, + "total_data_exchange": total_data_exchange, + "categories": categories_dict, + "support_tickets": support_tickets, + "datasets": self.get_paginated_response(datasets_serializer.data).data, + } + return Response(data, status=status.HTTP_200_OK) + + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response({}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class DatasetV2ViewSet(GenericViewSet): + """ + ViewSet for DatasetV2 model for create, update, detail/list view, & delete endpoints. + + **Context** + ``DatasetV2`` + An instance of :model:`datahub_datasetv2` + + **Serializer** + ``DatasetV2Serializer`` + :serializer:`datahub.serializer.DatasetV2Serializer` + + **Authorization** + ``ROLE`` only authenticated users/participants with following roles are allowed to make a POST request to this endpoint. + :role: `datahub_admin` (:role_id: `1`) + :role: `datahub_participant_root` (:role_id: `3`) + """ + + serializer_class = DatasetV2Serializer + queryset = DatasetV2.objects.prefetch_related('dataset_cat_map', 'dataset_cat_map__sub_category', + 'dataset_cat_map__sub_category__category').all() + pagination_class = CustomPagination + + @action(detail=False, methods=["post"]) + def validate_dataset(self, request, *args, **kwargs): + """ + ``POST`` method Endpoint: POST method to check the validation of dataset name and dataset description. [see here][ref] + + **Endpoint** + [ref]: /datahub/dataset/v2/dataset_validation/ + """ + serializer = DatasetV2Validation( + data=request.data, + context={ + "request_method": request.method, + "dataset_exists": request.query_params.get("dataset_exists"), + "queryset": self.queryset, + }, + ) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + return Response(serializer.data, status=status.HTTP_200_OK) + + @action(detail=False, methods=["post", "delete"]) + def temp_datasets(self, request, *args, **kwargs): + """ + ``POST`` method Endpoint: POST method to save the datasets in a temporary location with + under a newly created dataset name & source_file directory. + ``DELETE`` method Endpoint: DELETE method to delete the dataset named directory containing + the datasets. [see here][ref] + + **Endpoint** + [ref]: /datahub/dataset/v2/temp_datasets/ + """ + try: + files = request.FILES.getlist("datasets") + + if request.method == "POST": + """Create a temporary directory containing dataset files uploaded as source. + ``Example:`` + Create below directories with dataset files uploaded + /temp//file/ + """ + # serializer = DatasetV2TempFileSerializer(data=request.data, context={"request_method": request.method}) + serializer = DatasetV2TempFileSerializer( + data=request.data, + context={ + "request_method": request.method, + "dataset_exists": request.query_params.get("dataset_exists"), + "queryset": self.queryset, + }, + ) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + directory_created = file_operations.create_directory( + settings.TEMP_DATASET_URL, + [ + serializer.data.get("dataset_name"), + serializer.data.get("source"), + ], + ) + + files_saved = [] + for file in files: + file_operations.file_save(file, file.name, directory_created) + files_saved.append(file.name) + + data = {"datasets": files_saved} + data.update(serializer.data) + return Response(data, status=status.HTTP_201_CREATED) + + elif request.method == "DELETE": + """ + Delete the temporary directory containing datasets created by the POST endpoint + with the dataset files uploaded as source. + ``Example:`` + Delete the below directory: + /temp// + """ + serializer = DatasetV2TempFileSerializer( + data=request.data, + context={ + "request_method": request.method, + "query_params": request.query_params.get("delete_dir"), + }, + ) + + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + directory = string_functions.format_dir_name( + settings.TEMP_DATASET_URL, [request.data.get("dataset_name")] + ) + + """Delete directory temp directory as requested""" + if request.query_params.get("delete_dir") and os.path.exists(directory): + shutil.rmtree(directory) + LOGGER.info(f"Deleting directory: {directory}") + data = {request.data.get("dataset_name"): "Dataset not created"} + return Response(data, status=status.HTTP_204_NO_CONTENT) + + elif not request.query_params.get("delete_dir"): + """Delete a single file as requested""" + file_name = request.data.get("file_name") + file_path = os.path.join(directory, request.data.get("source"), file_name) + if os.path.exists(file_path): + os.remove(file_path) + LOGGER.info(f"Deleting file: {file_name}") + data = {file_name: "File deleted"} + return Response(data, status=status.HTTP_204_NO_CONTENT) + + return Response(status=status.HTTP_204_NO_CONTENT) + + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=False, methods=["get"]) + def get_dataset_files(self, request, *args, **kwargs): + """ + Get list of dataset files from temporary location. + """ + try: + # files_list = file_operations.get_csv_or_xls_files_from_directory(settings.TEMP_DATASET_URL + request.query_params.get(Constants.DATASET_NAME)) + dataset = request.data.get("dataset") + queryset = DatasetV2File.objects.filter(dataset=dataset) + serializer = DatasetFileV2NewSerializer(queryset, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + except Exception as error: + return Response(f"No such dataset created {error}", status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["post"]) + def get_dataset_file_columns(self, request, *args, **kwargs): + """ + To retrieve the list of columns of a dataset file from temporary location + """ + try: + dataset_file = DatasetV2File.objects.get(id=request.data.get("id")) + file_path = str(dataset_file.file) + if file_path.endswith(".xlsx") or file_path.endswith(".xls"): + df = pd.read_excel(os.path.join(settings.DATASET_FILES_URL, file_path), index_col=None, nrows=1) + else: + df = pd.read_csv(os.path.join(settings.DATASET_FILES_URL, file_path), index_col=False, nrows=1) + df.columns = df.columns.astype(str) + result = df.columns.tolist() + return Response(result, status=status.HTTP_200_OK) + except Exception as error: + LOGGER.error(f"Cannot get the columns of the selected file: {error}") + return Response( + f"Cannot get the columns of the selected file: {error}", + status=status.HTTP_400_BAD_REQUEST, + ) + + @action(detail=False, methods=["post"]) + def standardise(self, request, *args, **kwargs): + """ + Method to standardise a dataset and generate a file along with it. + """ + + # 1. Take the standardisation configuration variables. + try: + standardisation_configuration = request.data.get("standardisation_configuration") + mask_columns = request.data.get("mask_columns") + file_path = request.data.get("file_path") + is_standardised = request.data.get("is_standardised", None) + + if is_standardised: + file_path = file_path.replace("/standardised", "/datasets") + + if file_path.endswith(".xlsx") or file_path.endswith(".xls"): + df = pd.read_excel(os.path.join(settings.DATASET_FILES_URL, file_path), index_col=None) + else: + df = pd.read_csv(os.path.join(settings.DATASET_FILES_URL, file_path), index_col=False) + + df["status"] = True + df.loc[df["status"] == True, mask_columns] = "######" + # df[mask_columns] = df[mask_columns].mask(True) + del df["status"] + # print() + df.rename(columns=standardisation_configuration, inplace=True) + df.columns = df.columns.astype(str) + file_dir = file_path.split("/") + standardised_dir_path = "/".join(file_dir[-3:-1]) + file_name = file_dir[-1] + if not os.path.exists(os.path.join(settings.TEMP_STANDARDISED_DIR, standardised_dir_path)): + os.makedirs(os.path.join(settings.TEMP_STANDARDISED_DIR, standardised_dir_path)) + # print(df) + if file_name.endswith(".csv"): + df.to_csv( + os.path.join(settings.TEMP_STANDARDISED_DIR, standardised_dir_path, file_name) + ) # type: ignore + else: + df.to_excel( + os.path.join(settings.TEMP_STANDARDISED_DIR, standardised_dir_path, file_name) + ) # type: ignore + return Response( + {"standardised_file_path": f"{standardised_dir_path}/{file_name}"}, + status=status.HTTP_200_OK, + ) + + except Exception as error: + LOGGER.error(f"Could not standardise {error}") + return Response(error, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["get", "post"]) + def category(self, request, *args, **kwargs): + """ + ``GET`` method: GET method to retrieve the dataset category & sub categories from JSON file obj + ``POST`` method: POST method to create and/or edit the dataset categories & + sub categories and finally write it to JSON file obj. [see here][ref] + + **Endpoint** + [ref]: /datahub/dataset/v2/category/ + [JSON File Object]: "/categories.json" + """ + if request.method == "GET": + try: + with open(Constants.CATEGORIES_FILE, "r") as json_obj: + data = json.loads(json_obj.read()) + return Response(data, status=status.HTTP_200_OK) + except Exception as error: + LOGGER.error(error, exc_info=True) + raise custom_exceptions.NotFoundException(detail="Categories not found") + elif request.method == "POST": + try: + data = request.data + with open(Constants.CATEGORIES_FILE, "w+", encoding="utf8") as json_obj: + json.dump(data, json_obj, ensure_ascii=False) + LOGGER.info(f"Updated Categories: {Constants.CATEGORIES_FILE}") + return Response(data, status=status.HTTP_201_CREATED) + except Exception as error: + LOGGER.error(error, exc_info=True) + raise exceptions.InternalServerError("Internal Server Error") + + def create(self, request, *args, **kwargs): + """ + ``POST`` method Endpoint: create action to save the Dataset's Meta data + with datasets sent through POST request. [see here][ref]. + + **Endpoint** + [ref]: /datahub/dataset/v2/ + """ + try: + serializer = self.get_serializer( + data=request.data, + context={ + "standardisation_template": request.data.get("standardisation_template"), + "standardisation_config": request.data.get("standardisation_config"), + }, + ) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @authenticate_user(model=DatasetV2) + def update(self, request, pk, *args, **kwargs): + """ + ``PUT`` method: PUT method to edit or update the dataset (DatasetV2) and its files (DatasetV2File). [see here][ref] + + **Endpoint** + [ref]: /datahub/dataset/v2/ + """ + # setattr(request.data, "_mutable", True) + try: + data = request.data.copy() + to_delete = ast.literal_eval(data.get("deleted", "[]")) + sub_categories_map = data.pop("sub_categories_map") + self.dataset_files(data, to_delete) + datasetv2 = self.get_object() + serializer = self.get_serializer( + datasetv2, + data=data, + partial=True, + context={ + "standardisation_template": request.data.get("standardisation_template"), + "standardisation_config": request.data.get("standardisation_config"), + }, + ) + serializer.is_valid(raise_exception=True) + a = DatasetSubCategoryMap.objects.filter(dataset_id=datasetv2).delete() + serializer.save() + sub_categories_map = json.loads(sub_categories_map[0]) if c else [] + dataset_sub_cat_instances = [ + DatasetSubCategoryMap(dataset=datasetv2, sub_category=SubCategory.objects.get(id=sub_cat) + ) for sub_cat in sub_categories_map] + + DatasetSubCategoryMap.objects.bulk_create(dataset_sub_cat_instances) + + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + # not being used + @action(detail=False, methods=["delete"]) + def dataset_files(self, request, id=[]): + """ + ``DELETE`` method: DELETE method to delete the dataset files (DatasetV2File) referenced by DatasetV2 model. [see here][ref] + + **Endpoint** + [ref]: /datahub/dataset/v2/dataset_files/ + """ + ids = {} + for file_id in id: + dataset_file = DatasetV2File.objects.filter(id=int(file_id)) + if dataset_file.exists(): + LOGGER.info(f"Deleting file: {dataset_file[0].id}") + file_path = os.path.join("media", str(dataset_file[0].file)) + if os.path.exists(file_path): + os.remove(file_path) + dataset_file.delete() + ids[file_id] = "File deleted" + return Response(ids, status=status.HTTP_204_NO_CONTENT) + + def list(self, request, *args, **kwargs): + """ + ``GET`` method Endpoint: list action to view the list of Datasets via GET request. [see here][ref]. + + **Endpoint** + [ref]: /datahub/dataset/v2/ + """ + queryset = self.get_queryset() + # serializer = self.get_serializer(queryset, many=True) + # return Response(serializer.data, status=status.HTTP_200_OK) + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + return Response([], status=status.HTTP_404_NOT_FOUND) + + def retrieve(self, request, pk=None, *args, **kwargs): + """ + ``GET`` method Endpoint: retrieve action for the detail view of Dataset via GET request + Returns dataset object view with content of XLX/XLSX file and file URLS. [see here][ref]. + + **Endpoint** + [ref]: /datahub/dataset/v2// + """ + user_map = request.GET.get("user_map") + type = request.GET.get("type", None) + obj = self.get_object() + serializer = self.get_serializer(obj).data + dataset_file_obj = DatasetV2File.objects.prefetch_related("dataset_v2_file").filter(dataset_id=obj.id) + data = [] + for file in dataset_file_obj: + path_ = os.path.join(settings.DATASET_FILES_URL, str(file.standardised_file)) + file_path = {} + file_path["id"] = file.id + file_path["content"] = read_contents_from_csv_or_xlsx_file( + os.path.join(settings.DATASET_FILES_URL, str(file.standardised_file)), file.standardised_configuration + ) + file_path["file"] = path_ + file_path["source"] = file.source + file_path["file_size"] = file.file_size + file_path["accessibility"] = file.accessibility + file_path["standardised_file"] = os.path.join(settings.DATASET_FILES_URL, str(file.standardised_file)) + file_path["standardisation_config"] = file.standardised_configuration + type_filter = type if type == "api" else "dataset_file" + queryset = file.dataset_v2_file.filter(type=type_filter) + if user_map: + queryset = queryset.filter(user_organization_map=user_map) + usage_policy = UsagePolicyDetailSerializer( + queryset.order_by("-updated_at").all(), + many=True + ).data if queryset.exists() else [] + file_path["usage_policy"] = usage_policy + data.append(file_path) + + serializer["datasets"] = data + return Response(serializer, status=status.HTTP_200_OK) + + @action(detail=False, methods=["post"]) + def dataset_filters(self, request, *args, **kwargs): + """This function get the filter args in body. based on the filter args orm filters the data.""" + data = request.data + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + categories = data.pop(Constants.CATEGORY, None) + user_id = data.pop(Constants.USER_ID, "") + on_boarded_by = data.pop("on_boarded_by", "") + exclude_filters, filters = {}, {} + if others: + exclude_filters = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + else: + filters = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + try: + data = ( + DatasetV2.objects.select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ).prefetch_related( + 'dataset_cat_map') + .filter(**data, **filters) + .exclude(is_temp=True) + .exclude(**exclude_filters) + .order_by(Constants.UPDATED_AT) + .reverse() + .all() + ) + # if categories is not None: + # data = data.filter( + # reduce( + # operator.or_, + # (Q(category__contains=cat) for cat in categories), + # ) + # ) + if on_boarded_by: + data = ( + data.filter(user_map__user__on_boarded_by=user_id) + if others + else data.filter(user_map__user_id=user_id) + ) + else: + user_onboarded_by = User.objects.get(id=user_id).on_boarded_by + if user_onboarded_by: + data = ( + data.filter( + Q(user_map__user__on_boarded_by=user_onboarded_by.id) + | Q(user_map__user_id=user_onboarded_by.id) + ) + if others + else data.filter(user_map__user_id=user_id) + ) + else: + data = ( + data.filter(user_map__user__on_boarded_by=None).exclude(user_map__user__role_id=6) + if others + else data + ) + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error, exc_info=True) + return Response(f"Invalid filter fields: {list(request.data.keys())}", status=500) + page = self.paginate_queryset(data) + participant_serializer = DatahubDatasetsV2Serializer(page, many=True) + return self.get_paginated_response(participant_serializer.data) + + @action(detail=False, methods=["post"]) + @http_request_mutation + def filters_data(self, request, *args, **kwargs): + """This function provides the filters data""" + data = request.META + org_id = data.pop(Constants.ORG_ID, "") + others = data.pop(Constants.OTHERS, "") + user_id = data.pop(Constants.USER_ID, "") + + org_id = request.META.pop(Constants.ORG_ID, "") + others = request.META.pop(Constants.OTHERS, "") + user_id = request.META.pop(Constants.USER_ID, "") + + exclude, filters = {}, {} + if others: + exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + # filters = {Constants.APPROVAL_STATUS: Constants.APPROVED} + else: + filters = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + try: + geography = ( + DatasetV2.objects.values_list(Constants.GEOGRAPHY, flat=True) + .filter(**filters) + .exclude(geography="null") + .exclude(geography__isnull=True) + .exclude(geography="") + .exclude(is_temp=True, **exclude) + .all() + .distinct() + ) + # crop_detail = ( + # Datasets.objects.values_list(Constants.CROP_DETAIL, flat=True) + # .filter(status=True, **filters) + # .exclude(crop_detail="null") + # .exclude(crop_detail__isnull=True) + # .exclude(crop_detail="") + # .exclude(**exclude) + # .all() + # .distinct() + # ) + if os.path.exists(Constants.CATEGORIES_FILE): + with open(Constants.CATEGORIES_FILE, "r") as json_obj: + category_detail = json.loads(json_obj.read()) + else: + category_detail = [] + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + return Response(f"Invalid filter fields: {list(request.data.keys())}", status=500) + return Response({"geography": geography, "category_detail": category_detail}, status=200) + + # @action(detail=False, methods=["post"]) + # def search_datasets(self, request, *args, **kwargs): + # data = request.data + # org_id = data.pop(Constants.ORG_ID, "") + # others = data.pop(Constants.OTHERS, "") + # user_id = data.pop(Constants.USER_ID, "") + # search_pattern = data.pop(Constants.SEARCH_PATTERNS, "") + # exclude, filters = {}, {} + + # if others: + # exclude = {Constants.USER_MAP_ORGANIZATION: org_id} if org_id else {} + # filters = {Constants.NAME_ICONTAINS: search_pattern} if search_pattern else {} + # else: + # filters = ( + # { + # Constants.USER_MAP_ORGANIZATION: org_id, + # Constants.NAME_ICONTAINS: search_pattern, + # } + # if org_id + # else {} + # ) + # try: + # data = ( + # DatasetV2.objects.select_related( + # Constants.USER_MAP, + # Constants.USER_MAP_USER, + # Constants.USER_MAP_ORGANIZATION, + # ) + # .filter(user_map__user__status=True, status=True, **data, **filters) + # .exclude(**exclude) + # .order_by(Constants.UPDATED_AT) + # .reverse() + # .all() + # ) + # page = self.paginate_queryset(data) + # participant_serializer = DatahubDatasetsV2Serializer(page, many=True) + # return self.get_paginated_response(participant_serializer.data) + # except Exception as error: # type: ignore + # LOGGER.error("Error while filtering the datasets. ERROR: %s", error) + # return Response( + # f"Invalid filter fields: {list(request.data.keys())}", + # status=status.HTTP_500_INTERNAL_SERVER_ERROR, + # ) + + @authenticate_user(model=DatasetV2File) + def destroy(self, request, pk, *args, **kwargs): + """ + ``DELETE`` method: DELETE method to delete the DatasetV2 instance and its reference DatasetV2File instances, + along with dataset files stored at the URL. [see here][ref] + + **Endpoint** + [ref]: /datahub/dataset/v2/ + """ + try: + dataset_obj = self.get_object() + if dataset_obj: + dataset_files = DatasetV2File.objects.filter(dataset_id=dataset_obj.id) + dataset_dir = os.path.join(settings.DATASET_FILES_URL, str(dataset_obj.name)) + + if os.path.exists(dataset_dir): + shutil.rmtree(dataset_dir) + LOGGER.info(f"Deleting file: {dataset_dir}") + + # delete DatasetV2File & DatasetV2 instances + LOGGER.info(f"Deleting dataset obj: {dataset_obj}") + dataset_files.delete() + dataset_obj.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + +class DatasetV2ViewSetOps(GenericViewSet): + """ + A viewset for performing operations on datasets with Excel files. + + This viewset supports the following actions: + + - `dataset_names`: Returns the names of all datasets that have at least one Excel file. + - `dataset_file_names`: Given two dataset names, returns the names of all Excel files associated with each dataset. + - `dataset_col_names`: Given the paths to two Excel files, returns the column names of each file as a response. + - `dataset_join_on_columns`: Given the paths to two Excel files and the names of two columns, returns a JSON response with the result of an inner join operation on the two files based on the selected columns. + """ + + serializer_class = DatasetV2Serializer + queryset = DatasetV2.objects.all() + pagination_class = CustomPagination + + @action(detail=False, methods=["get"]) + def datasets_names(self, request, *args, **kwargs): + try: + datasets_with_excel_files = ( + DatasetV2.objects.prefetch_related("datasets") + .select_related("user_map") + .filter( + Q(datasets__file__endswith=".xls") + | Q(datasets__file__endswith=".xlsx") + | Q(datasets__file__endswith=".csv") + ) + .filter(user_map__organization_id=request.GET.get("org_id"), is_temp=False) + .distinct() + .values("name", "id", org_name=F("user_map__organization__name")) + ) + return Response(datasets_with_excel_files, status=status.HTTP_200_OK) + except Exception as e: + error_message = f"An error occurred while fetching dataset names: {e}" + return Response({"error": error_message}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=False, methods=["post"]) + def datasets_file_names(self, request, *args, **kwargs): + dataset_ids = request.data.get("datasets") + user_map = request.data.get("user_map") + if dataset_ids: + try: + # Get list of files for each dataset + files = ( + DatasetV2File.objects.select_related("dataset_v2_file", "dataset") + .filter(dataset_id__in=dataset_ids) + .filter(Q(file__endswith=".xls") | Q(file__endswith=".xlsx") | Q(file__endswith=".csv")) + .filter( + Q(accessibility__in=["public", "registered"]) + | Q(dataset__user_map_id=user_map) + | Q(dataset_v2_file__approval_status="approved") + ) + .values( + "id", + "dataset", + "standardised_file", + dataset_name=F("dataset__name"), + ) + .distinct() + ) + files = [ + { + **row, + "file_name": row.get("standardised_file", "").split("/")[-1], + } + for row in files + ] + return Response(files, status=status.HTTP_200_OK) + + except Exception as e: + return Response({"error": str(e)}, status=status.HTTP_400_BAD_REQUEST) + else: + return Response([], status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["post"]) + def datasets_col_names(self, request, *args, **kwargs): + try: + file_paths = request.data.get("files") + result = {} + for file_path in file_paths: + path = file_path + file_path = unquote(file_path).replace("/media/", "") + if file_path.endswith(".xlsx") or file_path.endswith(".xls"): + df = pd.read_excel( + os.path.join(settings.DATASET_FILES_URL, file_path), + index_col=None, + nrows=3, + ) + else: + df = pd.read_csv( + os.path.join(settings.DATASET_FILES_URL, file_path), + index_col=False, + nrows=3, + ) + df = df.drop(df.filter(regex="Unnamed").columns, axis=1) + result[path] = df.columns.tolist() + result[Constants.ID] = DatasetV2File.objects.get(standardised_file=file_path).id + return Response(result, status=status.HTTP_200_OK) + except Exception as e: + return Response({"error": str(e)}, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["post"]) + def datasets_join_condition(self, request, *args, **kwargs): + try: + file_path1 = request.data.get("file_path1") + file_path2 = request.data.get("file_path2") + columns1 = request.data.get("columns1") + columns2 = request.data.get("columns2") + condition = request.data.get("condition") + + # Load the files into dataframes + if file_path1.endswith(".xlsx") or file_path1.endswith(".xls"): + df1 = pd.read_excel(os.path.join(settings.MEDIA_ROOT, file_path1), usecols=columns1) + else: + df1 = pd.read_csv(os.path.join(settings.MEDIA_ROOT, file_path1), usecols=columns1) + if file_path2.endswith(".xlsx") or file_path2.endswith(".xls"): + df2 = pd.read_excel(os.path.join(settings.MEDIA_ROOT, file_path2), usecols=columns2) + else: + df2 = pd.read_csv(os.path.join(settings.MEDIA_ROOT, file_path2), usecols=columns2) + # Join the dataframes + result = pd.merge( + df1, + df2, + how=request.data.get("how", "left"), + left_on=request.data.get("left_on"), + right_on=request.data.get("right_on"), + ) + + # Return the joined dataframe as JSON + return Response(result.to_json(orient="records", index=False), status=status.HTTP_200_OK) + + except Exception as e: + LOGGER.error(str(e), exc_info=True) + return Response({"error": str(e)}, status=500) + + @action(detail=False, methods=["get"]) + def organization(self, request, *args, **kwargs): + """GET method: query the list of Organization objects""" + on_boarded_by = request.GET.get("on_boarded_by", "") + user_id = request.GET.get("user_id", "") + try: + user_org_queryset = ( + UserOrganizationMap.objects.prefetch_related("user_org_map") + .select_related("organization", "user") + .annotate(dataset_count=Count("user_org_map__id")) + .values( + name=F("organization__name"), + org_id=F("organization_id"), + org_description=F("organization__org_description"), + ) + .filter(user__status=True, dataset_count__gt=0) + .all() + ) + if on_boarded_by: + user_org_queryset = user_org_queryset.filter( + Q(user__on_boarded_by=on_boarded_by) | Q(user_id=on_boarded_by) + ) + else: + user_onboarded_by = User.objects.get(id=user_id).on_boarded_by + if user_onboarded_by: + user_org_queryset = user_org_queryset.filter( + Q(user__on_boarded_by=user_onboarded_by.id) | Q(user__id=user_onboarded_by.id) + ) + else: + user_org_queryset = user_org_queryset.filter(user__on_boarded_by=None).exclude(user__role_id=6) + return Response(user_org_queryset, 200) + except Exception as e: + error_message = f"An error occurred while fetching Organization details: {e}" + return Response({"error": error_message}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class StandardisationTemplateView(GenericViewSet): + serializer_class = StandardisationTemplateViewSerializer + queryset = StandardisationTemplate.objects.all() + + def create(self, request, *args, **kwargs): + try: + serializer = self.get_serializer(data=request.data, many=True) + serializer.is_valid(raise_exception=True) + serializer.save() + LOGGER.info("Standardisation Template Created Successfully.") + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=False, methods=["put"]) + def update_standardisation_template(self, request, *args, **kwargs): + update_list = list() + create_list = list() + try: + for data in request.data: + if data.get(Constants.ID, None): + # Update + id = data.pop(Constants.ID) + instance = StandardisationTemplate.objects.get(id=id) + serializer = StandardisationTemplateUpdateSerializer(instance, data=data, partial=True) + serializer.is_valid(raise_exception=True) + update_list.append(StandardisationTemplate(id=id, **data)) + else: + # Create + create_list.append(data) + + create_serializer = self.get_serializer(data=create_list, many=True) + create_serializer.is_valid(raise_exception=True) + StandardisationTemplate.objects.bulk_update( + update_list, fields=["datapoint_category", "datapoint_attributes"] + ) + create_serializer.save() + return Response(status=status.HTTP_201_CREATED) + except Exception as error: + LOGGER.error("Issue while Updating Standardisation Template", exc_info=True) + return Response( + f"Issue while Updating Standardisation Template {error}", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + def destroy(self, request, *args, **kwargs): + instance = self.get_object() + instance.delete() + LOGGER.info(f"Deleted datapoint Category from standardisation template {instance.datapoint_category}") + return Response(status=status.HTTP_204_NO_CONTENT) + + def list(self, request, *args, **kwargs): + serializer = self.get_serializer(self.get_queryset(), many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + +class PolicyListAPIView(generics.ListCreateAPIView): + queryset = Policy.objects.all() + serializer_class = PolicySerializer + + +class PolicyDetailAPIView(generics.RetrieveUpdateDestroyAPIView): + queryset = Policy.objects.all() + serializer_class = PolicySerializer + + +class DatasetV2View(GenericViewSet): + queryset = DatasetV2.objects.all() + serializer_class = DatasetV2NewListSerializer + pagination_class = CustomPagination + + def create(self, request, *args, **kwargs): + try: + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save() + LOGGER.info("Dataset created Successfully.") + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def retrieve(self, request, *args, **kwargs): + serializer = DatasetV2DetailNewSerializer(instance=self.get_object()) + return Response(serializer.data, status=status.HTTP_200_OK) + + @authenticate_user(model=DatasetV2) + def update(self, request, *args, **kwargs): + # setattr(request.data, "_mutable", True) + try: + instance = self.get_object() + data = request.data.copy() + sub_categories_map = data.pop("sub_categories_map") + data["is_temp"] = False + serializer = self.get_serializer(instance, data=data, partial=True) + serializer.is_valid(raise_exception=True) + DatasetSubCategoryMap.objects.filter(dataset_id=instance).delete() + serializer.save() + # sub_categories_map = json.loads(sub_categories_map[0]) if c else [] + dataset_sub_cat_instances = [ + DatasetSubCategoryMap(dataset=instance, sub_category=SubCategory.objects.get(id=sub_cat) + ) for sub_cat in sub_categories_map] + + DatasetSubCategoryMap.objects.bulk_create(dataset_sub_cat_instances) + + return Response(serializer.data, status=status.HTTP_200_OK) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @authenticate_user(model=DatasetV2) + def destroy(self, request, *args, **kwargs): + try: + instance = self.get_object() + instance.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["post"]) + def requested_datasets(self, request, *args, **kwargs): + try: + user_map_id = request.data.get("user_map") + policy_type = request.data.get("type", None) + if policy_type == "api": + dataset_file_id = request.data.get("dataset_file") + requested_recieved = ( + UsagePolicy.objects.select_related( + "dataset_file", + "dataset_file__dataset", + "user_organization_map__organization", + ) + .filter(dataset_file__dataset__user_map_id=user_map_id, dataset_file_id=dataset_file_id) + .values( + "id", + "approval_status", + "accessibility_time", + "updated_at", + "created_at", + dataset_id=F("dataset_file__dataset_id"), + dataset_name=F("dataset_file__dataset__name"), + file_name=F("dataset_file__file"), + organization_name=F("user_organization_map__organization__name"), + organization_email=F("user_organization_map__organization__org_email"), + organization_phone_number=F("user_organization_map__organization__phone_number"), + ) + .order_by("-updated_at") + ) + response_data = [] + for values in requested_recieved: + org = { + "org_email": values["organization_email"], + "name": values["organization_name"], + "phone_number": values["organization_phone_number"], + } + values.pop("organization_email") + values.pop("organization_name") + values.pop("organization_phone_number") + values["file_name"] = values.get("file_name", "").split("/")[-1] + + values["organization"] = org + response_data.append(values) + return Response( + { + "recieved": response_data, + }, + 200, + ) + requested_sent = ( + UsagePolicy.objects.select_related( + "dataset_file", + "dataset_file__dataset", + "user_organization_map__organization", + ) + .filter(user_organization_map=user_map_id, type="dataset_file") + .values( + "approval_status", + "updated_at", + "accessibility_time", + "type", + dataset_id=F("dataset_file__dataset_id"), + dataset_name=F("dataset_file__dataset__name"), + file_name=F("dataset_file__file"), + organization_name=F("dataset_file__dataset__user_map__organization__name"), + organization_email=F("dataset_file__dataset__user_map__organization__org_email"), + ) + .order_by("-updated_at") + ) + + requested_recieved = ( + UsagePolicy.objects.select_related( + "dataset_file", + "dataset_file__dataset", + "user_organization_map__organization", + ) + .filter(dataset_file__dataset__user_map_id=user_map_id, type="dataset_file") + .values( + "id", + "approval_status", + "accessibility_time", + "updated_at", + "type", + dataset_id=F("dataset_file__dataset_id"), + dataset_name=F("dataset_file__dataset__name"), + file_name=F("dataset_file__file"), + organization_name=F("user_organization_map__organization__name"), + organization_email=F("user_organization_map__organization__org_email"), + ) + .order_by("-updated_at") + ) + return Response( + { + "sent": [ + { + **values, + "file_name": values.get("file_name", "").split("/")[-1], + } + for values in requested_sent + ], + "recieved": [ + { + **values, + "file_name": values.get("file_name", "").split("/")[-1], + } + for values in requested_recieved + ], + }, + 200, + ) + except Exception as error: + LOGGER.error("Issue while Retrive requeted data", exc_info=True) + return Response( + f"Issue while Retrive requeted data {error}", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + # def list(self, request, *args, **kwargs): + # page = self.paginate_queryset(self.queryset) + # serializer = self.get_serializer(page, many=True).exclude(is_temp = True) + # return self.get_paginated_response(serializer.data) + + @http_request_mutation + @action(detail=True, methods=["post"]) + def get_dashboard_chart_data(self, request, pk, *args, **kwargs): + try: + filters = True + role_id = request.META.get("role_id") + map_id = request.META.get("map_id") + dataset_file_obj = DatasetV2File.objects.get(id=pk) + dataset_file = str(dataset_file_obj.file) + if role_id != str(1): + if UsagePolicy.objects.filter( + user_organization_map=map_id, + dataset_file_id=pk, + approval_status="approved" + ).order_by("-updated_at").first(): + filters = True + elif DatasetV2File.objects.select_related("dataset").filter(id=pk, dataset__user_map_id=map_id).first(): + filters = True + else: + filters = False + # Create a dictionary mapping dataset types to dashboard generation functions + dataset_type_to_dashboard_function = { + "omfp": generate_omfp_dashboard, + "fsp": generate_fsp_dashboard, + "knfd": generate_knfd_dashboard, + "kiamis": "generate_kiamis_dashboard", + } + + # Determine the dataset type based on the filename + dataset_type = None + for key in dataset_type_to_dashboard_function: + if key in dataset_file.lower(): + dataset_type = key + break + + # If dataset_type is not found, return an error response + if dataset_type is None: + return Response( + "Requested resource is currently unavailable. Please try again later.", + status=status.HTTP_200_OK, + ) + + # Generate the base hash key + hash_key = generate_hash_key_for_dashboard( + dataset_type if role_id == str(1) else pk, + request.data, role_id, filters + ) + + # Check if the data is already cached + cache_data = cache.get(hash_key, {}) + if cache_data: + LOGGER.info("Dashboard details found in cache", exc_info=True) + return Response(cache_data, status=status.HTTP_200_OK) + + # Get the appropriate dashboard generation function + dashboard_generator = dataset_type_to_dashboard_function.get(dataset_type) + + if dashboard_generator and dataset_type != 'kiamis': + # Generate the dashboard data using the selected function + return dashboard_generator( + dataset_file if role_id != str(1) else self.get_consolidated_file(dataset_type), + request.data, hash_key, filters + ) + + # serializer = DatahubDatasetFileDashboardFilterSerializer(data=request.data) + # serializer.is_valid(raise_exception=True) + counties = [] + sub_counties = [] + gender = [] + value_chain = [] + + # if serializer.data.get("county"): + counties = request.data.get("county") + + # if serializer.data.get("sub_county"): + sub_counties = request.data.get("sub_county") + + # if serializer.data.get("gender"): + gender = request.data.get("gender") + + # if serializer.data.get("value_chain"): + value_chain = request.data.get("value_chain") + cols_to_read = ['Gender', 'Constituency', 'Millet', 'County', 'Sub County', 'Crop Production', + 'farmer_mobile_number', + 'Livestock Production', 'Ducks', 'Other Sheep', 'Total Area Irrigation', 'Family', + 'Ward', + 'Other Money Lenders', 'Micro-finance institution', 'Self (Salary or Savings)', + "Natural rivers and stream", "Water Pan", + 'NPK', 'Superphosphate', 'CAN', + 'Urea', 'Other', 'Do you insure your crops?', + 'Do you insure your farm buildings and other assets?', 'Other Dual Cattle', + 'Cross breed Cattle', 'Cattle boma', + 'Small East African Goats', 'Somali Goat', 'Other Goat', 'Chicken -Indigenous', + 'Chicken -Broilers', 'Chicken -Layers', 'Highest Level of Formal Education', + 'Maize food crop', "Beans", 'Cassava', 'Sorghum', 'Potatoes', 'Cowpeas'] + if role_id == str(1): + dataset_file = self.get_consolidated_file("kiamis") + try: + if dataset_file.endswith(".xlsx") or dataset_file.endswith(".xls"): + df = pd.read_excel(os.path.join(settings.DATASET_FILES_URL, dataset_file)) + elif dataset_file.endswith(".csv"): + df = pd.read_csv(os.path.join(settings.DATASET_FILES_URL, dataset_file), usecols=cols_to_read, + low_memory=False) + # df.columns = df.columns.str.strip() + else: + return Response( + "Unsupported file please use .xls or .csv.", + status=status.HTTP_400_BAD_REQUEST, + ) + df['Ducks'] = pd.to_numeric(df['Ducks'], errors='coerce') + df['Other Sheep'] = pd.to_numeric(df['Other Sheep'], errors='coerce') + df['Family'] = pd.to_numeric(df['Family'], errors='coerce') + df['Other Money Lenders'] = pd.to_numeric(df['Other Money Lenders'], errors='coerce') + df['Micro-finance institution'] = pd.to_numeric(df['Micro-finance institution'], errors='coerce') + df['Self (Salary or Savings)'] = pd.to_numeric(df['Self (Salary or Savings)'], errors='coerce') + df['Natural rivers and stream'] = pd.to_numeric(df['Natural rivers and stream'], errors='coerce') + df["Water Pan"] = pd.to_numeric(df["Water Pan"], errors='coerce') + df['Total Area Irrigation'] = pd.to_numeric(df['Total Area Irrigation'], errors='coerce') + df['NPK'] = pd.to_numeric(df['NPK'], errors='coerce') + df['Superphosphate'] = pd.to_numeric(df['Superphosphate'], errors='coerce') + df['CAN'] = pd.to_numeric(df['CAN'], errors='coerce') + df['Urea'] = pd.to_numeric(df['Urea'], errors='coerce') + df['Other'] = pd.to_numeric(df['Other'], errors='coerce') + df['Other Dual Cattle'] = pd.to_numeric(df['Other Dual Cattle'], errors='coerce') + df['Cross breed Cattle'] = pd.to_numeric(df['Cross breed Cattle'], errors='coerce') + df['Cattle boma'] = pd.to_numeric(df['Cattle boma'], errors='coerce') + df['Small East African Goats'] = pd.to_numeric(df['Small East African Goats'], errors='coerce') + df['Somali Goat'] = pd.to_numeric(df['Somali Goat'], errors='coerce') + df['Other Goat'] = pd.to_numeric(df['Other Goat'], errors='coerce') + df['Chicken -Indigenous'] = pd.to_numeric(df['Chicken -Indigenous'], errors='coerce') + df['Chicken -Broilers'] = pd.to_numeric(df['Chicken -Broilers'], errors='coerce') + df['Chicken -Layers'] = pd.to_numeric(df['Chicken -Layers'], errors='coerce') + df['Do you insure your crops?'] = pd.to_numeric(df['Do you insure your crops?'], errors='coerce') + df['Highest Level of Formal Education'] = pd.to_numeric(df['Highest Level of Formal Education'], + errors='coerce') + df['Do you insure your farm buildings and other assets?'] = pd.to_numeric( + df['Do you insure your farm buildings and other assets?'], errors='coerce') + + data = filter_dataframe_for_dashboard_counties( + df=df, + counties=counties if counties else [], + sub_counties=sub_counties if sub_counties else [], + gender=gender if gender else [], + value_chain=value_chain if value_chain else [], + hash_key=hash_key, + filters=filters + ) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response( + f"Something went wrong, please try again. {e}", + status=status.HTTP_400_BAD_REQUEST, + ) + return Response( + data, + status=status.HTTP_200_OK, + ) + + except DatasetV2File.DoesNotExist as e: + LOGGER.error(e, exc_info=True) + return Response( + "No dataset file for the provided id.", + status=status.HTTP_404_NOT_FOUND, + ) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response( + f"Something went wrong, please try again. {e}", + status=status.HTTP_400_BAD_REQUEST, + ) + + def get_consolidated_file(self, name): + consolidated_file = f"consolidated_{name}.csv" + dataframes = [] + thread_list = [] + combined_df = pd.DataFrame([]) + try: + if os.path.exists(os.path.join(settings.DATASET_FILES_URL, consolidated_file)): + LOGGER.info(f"{consolidated_file} file available") + return consolidated_file + else: + dataset_file_objects = ( + DatasetV2File.objects + .select_related("dataset") + .filter(dataset__name__icontains=name, file__iendswith=".csv") + .values_list('file', flat=True).distinct() # Flatten the list of values + ) + + def read_csv_file(file_path): + chunk_size = 50000 + chunk_df = pd.DataFrame([]) + chunks = 0 + try: + LOGGER.info(f"{file_path} Consolidation started") + for chunk in pd.read_csv(file_path, chunksize=chunk_size): + # Append the processed chunk to the combined DataFrame + chunk_df = pd.concat([chunk_df, chunk], ignore_index=True) + chunks = chunks + 1 + LOGGER.info(f"{file_path} Consolidated {chunks} chunks") + dataframes.append(chunk_df) + except Exception as e: + LOGGER.error(f"Error reading CSV file {file_path}", exc_info=True) + + for csv_file in dataset_file_objects: + file_path = os.path.join(settings.DATASET_FILES_URL, csv_file) + thread = threading.Thread(target=read_csv_file, args=(file_path,)) + thread_list.append(thread) + thread.start() + + # Wait for all threads to complete + for thread in thread_list: + thread.join() + combined_df = pd.concat(dataframes, ignore_index=True) + combined_df.to_csv(os.path.join(settings.DATASET_FILES_URL, consolidated_file), index=False) + LOGGER.info(f"{consolidated_file} file created") + return consolidated_file + except Exception as e: + LOGGER.error(f"Error occoured while creating {consolidated_file}", exc_info=True) + return Response( + "Requested resource is currently unavailable. Please try again later.", + status=500, + ) + + +class DatasetFileV2View(GenericViewSet): + queryset = DatasetV2File.objects.all() + serializer_class = DatasetFileV2NewSerializer + + def create(self, request, *args, **kwargs): + validity = check_file_name_length(incoming_file_name=request.data.get("file"), + accepted_file_name_size=NumericalConstants.FILE_NAME_LENGTH) + if not validity: + return Response( + {"message": f"File name should not be more than {NumericalConstants.FILE_NAME_LENGTH} characters."}, + status=status.HTTP_400_BAD_REQUEST, + ) + try: + serializer = self.get_serializer(data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + data = serializer.data + instance = DatasetV2File.objects.get(id=data.get("id")) + instance.standardised_file = instance.file # type: ignore + instance.file_size = os.path.getsize(os.path.join(settings.DATASET_FILES_URL, str(instance.file))) + instance.save() + LOGGER.info("Dataset created Successfully.") + data = DatasetFileV2NewSerializer(instance) + return Response(data.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @authenticate_user(model=DatasetV2File) + def update(self, request, *args, **kwargs): + # setattr(request.data, "_mutable", True) + try: + data = request.data + instance = self.get_object() + # Generate the file and write the path to standardised file. + standardised_configuration = request.data.get("standardised_configuration") + # mask_columns = request.data.get( + # "mask_columns", + # ) + config = request.data.get("config") + file_path = str(instance.file) + if standardised_configuration: + if file_path.endswith(".xlsx") or file_path.endswith(".xls"): + df = pd.read_excel(os.path.join(settings.DATASET_FILES_URL, file_path), index_col=None) + else: + df = pd.read_csv(os.path.join(settings.DATASET_FILES_URL, file_path), index_col=False) + + df.rename(columns=standardised_configuration, inplace=True) + df.columns = df.columns.astype(str) + df = df.drop(df.filter(regex="Unnamed").columns, axis=1) + + if not os.path.exists(os.path.join(settings.DATASET_FILES_URL, instance.dataset.name, instance.source)): + os.makedirs(os.path.join(settings.DATASET_FILES_URL, instance.dataset.name, instance.source)) + + file_name = os.path.basename(file_path).replace(".", "_standerdise.") + if file_path.endswith(".csv"): + df.to_csv( + os.path.join( + settings.DATASET_FILES_URL, + instance.dataset.name, + instance.source, + file_name, + ) + ) # type: ignore + else: + df.to_excel( + os.path.join( + settings.DATASET_FILES_URL, + instance.dataset.name, + instance.source, + file_name, + ) + ) # type: ignore + # data = request.data + standardised_file_path = os.path.join(instance.dataset.name, instance.source, file_name) + data["file_size"] = os.path.getsize( + os.path.join(settings.DATASET_FILES_URL, str(standardised_file_path))) + else: + file_name = os.path.basename(file_path) + standardised_file_path = os.path.join(instance.dataset.name, instance.source, file_name) + # data["file_size"] = os.path.getsize(os.path.join(settings.DATASET_FILES_URL, str(standardised_file_path))) + data["standardised_configuration"] = config + serializer = self.get_serializer(instance, data=data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + DatasetV2File.objects.filter(id=serializer.data.get("id")).update(standardised_file=standardised_file_path) + return Response(serializer.data, status=status.HTTP_200_OK) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def list(self, request, *args, **kwargs): + data = DatasetV2File.objects.filter(dataset=request.GET.get("dataset")).values("id", "file") + return Response(data, status=status.HTTP_200_OK) + + @authenticate_user(model=DatasetV2File) + def destroy(self, request, *args, **kwargs): + try: + dataset_file = self.get_object() + dataset_file.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + # @action(detail=False, methods=["put"]) + @authenticate_user(model=DatasetV2File) + def patch(self, request, *args, **kwargs): + try: + instance = self.get_object() + serializer = self.get_serializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=status.HTTP_200_OK) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + +class UsagePolicyListCreateView(generics.ListCreateAPIView): + queryset = UsagePolicy.objects.all() + serializer_class = UsagePolicySerializer + + +class UsagePolicyRetrieveUpdateDestroyView(generics.RetrieveUpdateDestroyAPIView): + queryset = UsagePolicy.objects.all() + serializer_class = UsageUpdatePolicySerializer + api_builder_serializer_class = APIBuilderSerializer + + @authenticate_user(model=UsagePolicy) + def patch(self, request, *args, **kwargs): + instance = self.get_object() + approval_status = request.data.get('approval_status') + policy_type = request.data.get('type', None) + instance.api_key = None + try: + if policy_type == 'api': + if approval_status == 'approved': + instance.api_key = generate_api_key() + serializer = self.api_builder_serializer_class(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=200) + + except ValidationError as e: + LOGGER.error(e, exc_info=True) + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class DatahubNewDashboard(GenericViewSet): + """Datahub Dashboard viewset""" + + pagination_class = CustomPagination + + def participant_metics(self, data): + on_boarded_by = data.get("onboarded_by") + role_id = data.get("role_id") + user_id = data.get("user_id") + result = {} + try: + if on_boarded_by != "None" or role_id == str(6): + result["participants_count"] = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter( + user__status=True, + user__on_boarded_by=on_boarded_by if on_boarded_by != "None" else user_id, + user__role=3, + user__approval_status=True, + ) + .count() + ) + elif role_id == str(1): + result["co_steward_count"] = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter(user__status=True, user__role=6) + .count() + ) + result["participants_count"] = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter( + user__status=True, + user__role=3, + user__on_boarded_by=None, + user__approval_status=True, + ) + .count() + ) + else: + result["participants_count"] = ( + UserOrganizationMap.objects.select_related(Constants.USER, Constants.ORGANIZATION) + .filter( + user__status=True, + user__role=3, + user__on_boarded_by=None, + user__approval_status=True, + ) + .count() + ) + LOGGER.info("Participants Metrics completed") + return result + except Exception as error: # type: ignore + LOGGER.error( + "Error while filtering the participants. ERROR: %s", + error, + exc_info=True, + ) + raise Exception(str(error)) + + def dataset_metrics(self, data, request): + on_boarded_by = data.get("onboarded_by") + role_id = data.get("role_id") + user_id = data.get("user_id") + user_org_map = data.get("map_id") + try: + query = ( + DatasetV2.objects.prefetch_related("datasets") + .select_related( + Constants.USER_MAP, + Constants.USER_MAP_USER, + Constants.USER_MAP_ORGANIZATION, + ) + .exclude(is_temp=True) + ) + if on_boarded_by != "None" or role_id == str(6): + query = query.filter( + Q(user_map__user__on_boarded_by=on_boarded_by if on_boarded_by != "None" else user_id) + | Q(user_map__user_id=on_boarded_by if on_boarded_by != "None" else user_id) + ) + else: + query = query.filter(user_map__user__on_boarded_by=None).exclude(user_map__user__role_id=6) + LOGGER.info("Datasets Metrics completed") + return query + except Exception as error: # type: ignore + LOGGER.error("Error while filtering the datasets. ERROR: %s", error, exc_info=True) + raise Exception(str(error)) + + def connector_metrics(self, data, dataset_query, request): + # on_boarded_by = data.get("onboarded_by") + # role_id = data.get("role_id") + user_id = data.get("user_id") + user_org_map = data.get("map_id") + my_dataset_used_in_connectors = ( + dataset_query.prefetch_related("datasets__right_dataset_file") + .values("datasets__right_dataset_file") + .filter(datasets__right_dataset_file__connectors__user_id=user_id) + .distinct() + .count() + + dataset_query.prefetch_related("datasets__left_dataset_file") + .values("datasets__left_dataset_file") + .filter(datasets__left_dataset_file__connectors__user_id=user_id) + .distinct() + .count() + ) + connectors_query = Connectors.objects.filter(user_id=user_id).all() + + other_datasets_used_in_my_connectors = ( + dataset_query.prefetch_related("datasets__right_dataset_file") + .select_related("datasets__right_dataset_file__connectors") + .filter(datasets__right_dataset_file__connectors__user_id=user_id) + .values("datasets__right_dataset_file") + .exclude(user_map_id=user_org_map) + .distinct() + .count() + ) + ( + dataset_query.prefetch_related("datasets__left_dataset_file") + .select_related("datasets__left_dataset_file__connectors") + .filter(datasets__left_dataset_file__connectors__user_id=user_id) + .values("datasets__left_dataset_file") + .exclude(user_map_id=user_org_map) + .distinct() + .count() + ) + return { + "total_connectors_count": connectors_query.count(), + "other_datasets_used_in_my_connectors": other_datasets_used_in_my_connectors, + "my_dataset_used_in_connectors": my_dataset_used_in_connectors, + "recent_connectors": ConnectorsListSerializer( + connectors_query.order_by("-updated_at")[0:3], many=True + ).data, + } + + @action(detail=False, methods=["get"]) + @http_request_mutation + def dashboard(self, request): + """Retrieve datahub dashboard details""" + data = request.META + try: + participant_metrics = self.participant_metics(data) + dataset_query = self.dataset_metrics(data, request) + # This will fetch connectors metrics + connector_metrics = self.connector_metrics(data, dataset_query, request) + if request.GET.get("my_org", False): + dataset_query = dataset_query.filter(user_map_id=data.get("map_id")) + dataset_file_metrics = ( + dataset_query.values("datasets__source") + .annotate( + dataset_count=Count("id", distinct=True), + file_count=Count("datasets__file", distinct=True), + total_size=Sum("datasets__file_size"), + ) + .filter(file_count__gt=0) + ) + + dataset_state_metrics = dataset_query.values(state_name=F("geography__state__name")).annotate( + dataset_count=Count("id", distinct=True) + ) + distinct_keys = ( + DatasetV2.objects.annotate( + key=Func( + "category", + function="JSONB_OBJECT_KEYS", + output_field=CharField(), + ) + ) + .values_list("key", flat=True) + .distinct() + ) + + # Iterate over the distinct keys and find the count for each key + dataset_category_metrics = {} + for key in distinct_keys: + dataset_count = dataset_query.filter(category__has_key=key).count() + if dataset_count: + dataset_category_metrics[key] = dataset_count + recent_datasets = DatasetV2ListNewSerializer(dataset_query.order_by("-updated_at")[0:3], many=True).data + data = { + "user": UserOrganizationMap.objects.select_related("user", "organization") + .filter(id=data.get("map_id")) + .values( + first_name=F("user__first_name"), + last_name=F("user__last_name"), + logo=Concat( + Value("media/"), + F("organization__logo"), + output_field=CharField(), + ), + org_email=F("organization__org_email"), + name=F("organization__name"), + ) + .first(), + "total_participants": participant_metrics, + "dataset_file_metrics": dataset_file_metrics, + "dataset_state_metrics": dataset_state_metrics, + "total_dataset_count": dataset_query.count(), + "dataset_category_metrics": dataset_category_metrics, + "recent_datasets": recent_datasets, + **connector_metrics, + } + return Response(data, status=status.HTTP_200_OK) + + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response({}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +# @http_request_mutation +class ResourceManagementViewSet(GenericViewSet): + """ + Resource Management viewset. + """ + + queryset = Resource.objects.prefetch_related().all() + serializer_class = ResourceSerializer + pagination_class = CustomPagination + + @http_request_mutation + @transaction.atomic + def create(self, request, *args, **kwargs): + try: + user_map = request.META.get("map_id") + # request.data._mutable = True + data = request.data.copy() + files = request.FILES.getlist('files') # 'files' is the key used in FormData + data["files"] = files + data["user_map"] = user_map + + serializer = self.get_serializer(data=data) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + LOGGER.error(e, exc_info=True) + return Response(e, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @transaction.atomic + def update(self, request, *args, **kwargs): + try: + instance = self.get_object() + data = request.data.copy() + sub_categories_map = data.pop("sub_categories_map") + serializer = self.get_serializer(instance, data=data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + resource_id = serializer.data.get("id") + ResourceSubCategoryMap.objects.filter(resource=instance).delete() + + sub_categories_map = json.loads(sub_categories_map[0]) if sub_categories_map else [] + resource_sub_cat_instances = [ + ResourceSubCategoryMap(resource=instance, sub_category=SubCategory.objects.get(id=sub_cat) + ) for sub_cat in sub_categories_map] + ResourceSubCategoryMap.objects.bulk_create(resource_sub_cat_instances) + + return Response(serializer.data, status=status.HTTP_200_OK) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @http_request_mutation + def list(self, request, *args, **kwargs): + try: + user_map = request.META.get("map_id") + if request.GET.get("others", None): + queryset = Resource.objects.exclude(user_map=user_map).order_by("-updated_at") + else: + queryset = Resource.objects.filter(user_map=user_map).order_by("-updated_at") + + page = self.paginate_queryset(queryset) + serializer = ResourceListSerializer(page, many=True) + return self.get_paginated_response(serializer.data) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def destroy(self, request, *args, **kwargs): + resource = self.get_object() + resource.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + + @http_request_mutation + def retrieve(self, request, *args, **kwargs): + user_map = request.META.get("map_id") + resource = self.get_object() + serializer = self.get_serializer(resource) + data = serializer.data.copy() + try: + if str(resource.user_map_id) == str(user_map): + resource_usage_policy = ( + ResourceUsagePolicy.objects.select_related( + "resource", + ) + .filter(resource=resource) + .values( + "id", + "approval_status", + "accessibility_time", + "updated_at", + "type", + "api_key", + "created_at", + "resource_id", + resource_title=F("resource__title"), + organization_name=F("user_organization_map__organization__name"), + organization_email=F("user_organization_map__organization__org_email"), + organization_phone_number=F("user_organization_map__organization__phone_number"), + ) + .order_by("-updated_at") + ) + # data["retrival"] = MessagesChunksRetriveSerializer(Messages.objects.filter(resource_id=resource.id).order_by("-created_at").all(), many=True).data + else: + resource_usage_policy = ( + ResourceUsagePolicy.objects.select_related( + "resource" + ) + .filter(user_organization_map_id=user_map, resource=resource) + .values( + "id", + "approval_status", + "updated_at", + "accessibility_time", + "type", + "resource_id", + "api_key", + organization_name=F("user_organization_map__organization__name"), + organization_email=F("user_organization_map__organization__org_email"), + organization_phone_number=F("user_organization_map__organization__phone_number"), + ) + .order_by("-updated_at") + ) + data["resource_usage_policy"] = resource_usage_policy + print(data.get(resource_usage_policy)) + return Response(data, status=status.HTTP_200_OK) + except Exception as error: + LOGGER.error("Issue while Retrive Resource details", exc_info=True) + return Response( + f"Issue while Retrive Retrive Resource details {error}", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + @http_request_mutation + @action(detail=False, methods=["post"]) + def resources_filter(self, request, *args, **kwargs): + try: + data = request.data + user_map = request.META.get("map_id") + categories = data.pop(Constants.CATEGORY, None) + others = data.pop(Constants.OTHERS, "") + filters = {key: value for key, value in data.items() if value} + query_set = self.get_queryset().filter(**filters).order_by("-updated_at") + if categories: + query_set = query_set.filter( + reduce( + operator.or_, + (Q(category__contains=cat) for cat in categories), + ) + ) + query_set = query_set.exclude(user_map=user_map) if others else query_set.filter( + user_map=user_map) + + page = self.paginate_queryset(query_set) + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=False, methods=["post"]) + def requested_resources(self, request, *args, **kwargs): + try: + user_map_id = request.data.get("user_map") + # policy_type = request.data.get("type", None) + # resource_id = request.data.get("resource") + requested_recieved = ( + ResourceUsagePolicy.objects.select_related( + "resource", + ) + .filter(resource__user_map_id=user_map_id) + .values( + "id", + "approval_status", + "accessibility_time", + "updated_at", + "type", + "created_at", + "resource_id", + resource_title=F("resource__title"), + organization_name=F("user_organization_map__organization__name"), + organization_email=F("user_organization_map__organization__org_email"), + organization_phone_number=F("user_organization_map__organization__phone_number"), + ) + .order_by("-updated_at") + ) + + requested_sent = ( + ResourceUsagePolicy.objects.select_related( + "resource" + ) + .filter(user_organization_map_id=user_map_id) + .values( + "id", + "approval_status", + "updated_at", + "accessibility_time", + "type", + "resource_id", + resource_title=F("resource__title"), + organization_name=F("resource__user_map__organization__name"), + organization_email=F("resource__user_map__organization__org_email"), + ) + .order_by("-updated_at") + ) + + return Response( + { + "sent": requested_sent, + "recieved": requested_recieved, + }, + 200, + ) + except Exception as error: + LOGGER.error("Issue while Retrive Resource requeted data", exc_info=True) + return Response( + f"Issue while Retrive Resource requeted data {error}", + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +class ResourceFileManagementViewSet(GenericViewSet): + """ + Resource File Management + """ + + queryset = ResourceFile.objects.all() + serializer_class = ResourceFileSerializer + + @http_request_mutation + @transaction.atomic + def create(self, request, *args, **kwargs): + try: + data = request.data.copy() + resource = data.get("resource") + if data.get("type") == "youtube": + youtube_urls_response = get_youtube_url(data.get("url")) + if youtube_urls_response.status_code == 400: + return youtube_urls_response + youtube_urls = youtube_urls_response.data + playlist_urls = [{"resource": resource, "type": "youtube", **row} for row in youtube_urls] + for row in playlist_urls: + serializer = self.get_serializer(data=row) + serializer.is_valid(raise_exception=True) + serializer.save() + LOGGER.info(f"Embeding creation started for youtube url: {row.get('url')}") + VectorDBBuilder.create_vector_db.delay(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED) + else: + serializer = self.get_serializer(data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + VectorDBBuilder.create_vector_db.delay(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @transaction.atomic + def destroy(self, request, *args, **kwargs): + instance = self.get_object() + resourcefile_id = instance.id + collections_to_delete = LangchainPgCollection.objects.filter(name=resourcefile_id) + collections_to_delete.delete() + instance.delete() + + return Response(status=status.HTTP_204_NO_CONTENT) + + @action(detail=False, methods=["post"]) + def resource_live_api_export(self, request): + """This is an API to fetch the data from an External API with an auth token + and store it in JSON format.""" + try: + url = request.data.get("url") + auth_type = request.data.get("auth_type") + title = request.data.get("title") + source = request.data.get("source") + file_name = request.data.get("file_name") + resource = request.data.get("resource") + if auth_type == 'NO_AUTH': + response = requests.get(url) + elif auth_type == 'API_KEY': + headers = {request.data.get( + "api_key_name"): request.data.get("api_key_value")} + response = requests.get(url, headers=headers) + elif auth_type == 'BEARER': + headers = {"Authorization": "Bearer " + + request.data.get("token")} + response = requests.get(url, headers=headers) + + # response = requests.get(url) + if response.status_code in [200, 201]: + try: + data = response.json() + except ValueError: + data = response.text + file_path = settings.RESOURCES_URL + f"file {str(uuid.uuid4())}.json" + format = "w" if os.path.exists(file_path) else "x" + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, format) as outfile: + if type(data) == list: + json.dump(data, outfile) + else: + outfile.write(json.dumps(data)) + if resource: + with open(file_path, "rb") as outfile: # Open the file in binary read mode + # Wrap the file content using Django's ContentFile + django_file = ContentFile(outfile.read(), + name=f"{file_name}.json") # You can give it any name you prefer + + # Prepare data for serializer + serializer_data = {"resource": resource, "type": "api", "file": django_file} + + # Initialize and validate serializer + serializer = ResourceFileSerializer(data=serializer_data) + serializer.is_valid(raise_exception=True) + serializer.save() + VectorDBBuilder.create_vector_db.delay(serializer.data) + return JsonResponse(serializer.data, status=status.HTTP_200_OK) + return Response(file_path) + LOGGER.error("Failed to fetch data from api") + return Response({"message": f"API Response: {response.json()}"}, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error( + f"Failed to fetch data from api ERROR: {e} and input fields: {request.data}") + return Response({"message": f"API Response: {e}"}, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["get"]) + def fetch_videos(self, request): + url = request.GET.get("url") + return get_youtube_url(url) + + def retrieve(self, request, pk): + """GET method: retrieve an object or instance of the Product model""" + team_member = self.get_object() + serializer = self.get_serializer(team_member) + # serializer.is_valid(raise_exception=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + +# +class CategoryViewSet(viewsets.ModelViewSet): + queryset = Category.objects.prefetch_related("subcategory_category").all() + serializer_class = CategorySerializer + + @action(detail=False, methods=["get"]) + def categories_and_sub_categories(self, request): + categories_with_subcategories = {} + # Retrieve all categories and their related subcategories + categories = Category.objects.all() + + for category in categories: + # Retrieve the names of all subcategories related to this category + subcategory_names = [sub_category.name for sub_category in + SubCategory.objects.filter(category=category).all()] + # Assign the list of subcategory names to the category name in the dictionary + categories_with_subcategories[category.name] = subcategory_names + + return Response(categories_with_subcategories, 200) + + +class SubCategoryViewSet(viewsets.ModelViewSet): + queryset = SubCategory.objects.all() + serializer_class = SubCategorySerializer + permission_classes = [] + + @action(detail=False, methods=["post"]) + def dump_categories(self, request): + data = request.data + for category_name, sub_categories in data.items(): + category, created = Category.objects.get_or_create(name=category_name) + print(category) + for sub_category_name in sub_categories: + SubCategory.objects.get_or_create(category=category, name=sub_category_name) + return Response("Data dumped") + + @action(detail=False, methods=["get"]) + @transaction.atomic + def dump_resource_categories_map(self, request): + # Parse the category JSON field + resources = Resource.objects.all() + for resource in resources: + categories = resource.category + for category_name, sub_category_names in categories.items(): + category = Category.objects.filter(name=category_name).first() + for sub_category_name in sub_category_names: + # Find the corresponding SubCategory instance + try: + sub_category = SubCategory.objects.filter(name=sub_category_name, category=category).first() + print(sub_category) + print(resource) + if sub_category: + ResourceSubCategoryMap.objects.get_or_create( + sub_category=sub_category, + resource=resource + ) + except SubCategory.DoesNotExist: + print(f"SubCategory '{sub_category_name}' does not exist.") + return Response("Data dumped") + + @action(detail=False, methods=["get"]) + @transaction.atomic + def dump_dataset_map(self, request): + # Parse the category JSON field + resources = DatasetV2.objects.all() + for resource in resources: + categories = resource.category + for category_name, sub_category_names in categories.items(): + category = Category.objects.filter(name=category_name).first() + for sub_category_name in sub_category_names: + # Find the corresponding SubCategory instance + try: + sub_category = SubCategory.objects.filter(name=sub_category_name, category=category).first() + if sub_category: + DatasetSubCategoryMap.objects.get_or_create( + sub_category=sub_category, + dataset=resource + ) + except SubCategory.DoesNotExist: + print(f"SubCategory '{sub_category_name}' does not exist.") + return Response("Data dumped") + + +class EmbeddingsViewSet(viewsets.ModelViewSet): + queryset = LangchainPgEmbedding.objects.all() + serializer_class = LangChainEmbeddingsSerializer + lookup_field = 'uuid' # Specify the UUID field as the lookup field + + @action(detail=False, methods=['get']) + def embeddings_and_chunks(self, request): + embeddings = [] + collection_id = request.GET.get("resource_file") + collection = LangchainPgCollection.objects.filter(name=str(collection_id)).first() + if collection: + embeddings = LangchainPgEmbedding.objects.filter(collection_id=collection.uuid).values("document") + return Response(embeddings) + + @action(detail=False, methods=['post']) + def get_embeddings(self, request): + # Use the 'uuid' field to look up the instance + # instance = self.get_object() + uuid = request.data.get("uuid") + data = LangchainPgEmbedding.objects.filter(uuid=uuid).values("embedding", "document") + # print(data) + # import pdb; pdb.set_trace() + # serializer = self.get_serializer(data) + return Response(data) + + @http_request_mutation + @action(detail=False, methods=['post']) + def chat_api(self, request): + map_id = request.META.get("map_id") + user_id = request.META.get("user_id") + data = request.data + query = request.data.get("query") + resource_id = request.data.get("resource") + try: + user_name = User.objects.get(id=user_id).first_name + history = Messages.objects.filter(user_map=map_id).order_by("-created_at") + history = history.filter(resource_id=resource_id).first() if resource_id else history.first() + + # print(chat_history) + # chat_history = history.condensed_question if history else "" + summary, chunks, condensed_question, prompt_usage = VectorDBBuilder.get_input_embeddings(query, user_name, + resource_id, + history) + data = {"user_map": UserOrganizationMap.objects.get(id=map_id).id, "resource": resource_id, "query": query, + "query_response": summary, "condensed_question": condensed_question, "prompt_usage": prompt_usage} + messages_serializer = MessagesSerializer(data=data) + messages_serializer.is_valid(raise_exception=True) + message_instance = messages_serializer.save() # This returns the Messages model instance + data = messages_serializer.data + if chunks: + message_instance.retrieved_chunks.set(chunks.values_list("uuid", flat=True)) + return Response(data) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(f"Error During the execution: {str(e)}", status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @http_request_mutation + @action(detail=False, methods=['post']) + def chat_histroy(self, request): + try: + map_id = request.META.get("map_id") + resource_id = request.data.get("resource") + history = Messages.objects.filter(user_map=map_id, bot_type="vistaar").order_by("created_at") + if resource_id: + history = history.filter(resource_id=resource_id).all() + else: + history = history.filter(resource_id__isnull=True).all() + total = len(history) + slice = 0 if total <= 10 else total - 10 + history = history[slice:total] + messages_serializer = MessagesRetriveSerializer(history, many=True) + return Response(messages_serializer.data) + except ValidationError as e: + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + LOGGER.error(e, exc_info=True) + return Response(str(e), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=False, methods=['get']) + def dump_embeddings(self, request): + # queryset = ResourceFile.objects.all() + from django.db import models + from django.db.models.functions import Cast + + queryset = ResourceFile.objects.exclude( + id__in=Subquery( + LangchainPgCollection.objects.annotate( + uuid_name=Cast('name', models.UUIDField()) + ).values('uuid_name') + ) + ).all() + serializer = ResourceFileSerializer(queryset, many=True) + data = serializer.data + total_files = len(data) + count = 0 + for row in data: + count += 1 + VectorDBBuilder.create_vector_db(row) + print(f"resource {row} is completed") + print(f"{count} completed out of {total_files}") + return Response("embeddings created for all the files") + + +class ResourceUsagePolicyListCreateView(generics.ListCreateAPIView): + queryset = ResourceUsagePolicy.objects.all() + serializer_class = ResourceUsagePolicySerializer + + +class ResourceUsagePolicyRetrieveUpdateDestroyView(generics.RetrieveUpdateDestroyAPIView): + queryset = ResourceUsagePolicy.objects.all() + serializer_class = ResourceUsagePolicySerializer + api_builder_serializer_class = ResourceAPIBuilderSerializer + + @authenticate_user(model=ResourceUsagePolicy) + def patch(self, request, *args, **kwargs): + # import pdb;pdb.set_trace() + instance = self.get_object() + approval_status = request.data.get('approval_status') + policy_type = request.data.get('type', None) + instance.api_key = None + try: + if approval_status == 'approved': + instance.api_key = generate_api_key() + serializer = self.api_builder_serializer_class(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=200) + + except ValidationError as e: + LOGGER.error(e, exc_info=True) + return Response(e.detail, status=status.HTTP_400_BAD_REQUEST) + + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class MessagesViewSet(generics.RetrieveUpdateDestroyAPIView): + queryset = Messages.objects.all() + serializer_class = MessagesSerializer + + def delete(self, request, *args, **kwargs): + return Response("You don't have access to delete the chat history", status=status.HTTP_400_BAD_REQUEST) + + def patch(self, request, *args, **kwargs): + try: + instance = self.get_object() + serializer = MessagesRetriveSerializer(instance, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=status.HTTP_200_OK) + except Exception as error: + LOGGER.error(error, exc_info=True) + return Response(str(error), status=status.HTTP_400_BAD_REQUEST) + + +class MessagesCreateViewSet(generics.ListCreateAPIView): + queryset = Messages.objects.all() + serializer_class = MessagesSerializer + pagination_class = CustomPagination + + @http_request_mutation + def list(self, request, *args, **kwargs): + resource_id = request.GET.get("resource") + user_map = request.META.get("map_id") + if resource_id: + queryset = Messages.objects.filter(resource_id=resource_id).order_by("-created_at").all() + else: + queryset = Messages.objects.filter(user_map=user_map).order_by("-created_at").all() + page = self.paginate_queryset(queryset) + serializer = MessagesChunksRetriveSerializer(page, many=True) + return self.get_paginated_response(serializer.data) \ No newline at end of file diff --git a/utils/authentication_services.py b/utils/authentication_services.py index ac0be87b..c82dd1b5 100644 --- a/utils/authentication_services.py +++ b/utils/authentication_services.py @@ -6,8 +6,8 @@ from connectors.models import Connectors from core.constants import Constants -from datahub.models import ( - Datasets, +from datasets.models import Datasets +from participant.models import ( DatasetV2, DatasetV2File, Organization, diff --git a/utils/embeddings_creation.py b/utils/embeddings_creation.py index cc54370a..36681f4b 100644 --- a/utils/embeddings_creation.py +++ b/utils/embeddings_creation.py @@ -66,7 +66,7 @@ from core import settings from core.constants import Constants -from datahub.models import LangchainPgCollection, LangchainPgEmbedding, ResourceFile +from participant.models import LangchainPgCollection, LangchainPgEmbedding, ResourceFile # from openai import OpenAI LOGGING = logging.getLogger(__name__) From c538f06ba2e76407d57f4035be27a81560ba04f5 Mon Sep 17 00:00:00 2001 From: Jai Date: Mon, 4 Mar 2024 15:23:03 +0530 Subject: [PATCH 2/5] UNSTABLE -> Moving Datahub to Participant / Moving Dataset to its own app --- core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/settings.py b/core/settings.py index 070112de..d2b24549 100644 --- a/core/settings.py +++ b/core/settings.py @@ -130,7 +130,7 @@ } # Password validation -# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-v\alidators +# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { From a9356b30fc540f3e3ad0d45e9fc7b86c70d41ca4 Mon Sep 17 00:00:00 2001 From: Jai Date: Mon, 4 Mar 2024 15:31:59 +0530 Subject: [PATCH 3/5] UNSTABLE -> Moving Datahub to Participant / Moving Dataset to its own app --- accounts/tests/test_urls.py | 2 +- accounts/views.py | 2 +- datahub/tests/test_api_builder.py | 2 +- datahub/tests/test_datahub_views.py | 2 +- microsite/tests/test_home_view.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/accounts/tests/test_urls.py b/accounts/tests/test_urls.py index 2b466afe..8fa8a61d 100644 --- a/accounts/tests/test_urls.py +++ b/accounts/tests/test_urls.py @@ -1,7 +1,7 @@ from django.test import SimpleTestCase from django.urls import resolve, reverse from accounts.views import RegisterViewset -from datahub.views import ParticipantViewSet +from participant.views import ParticipantViewSet from django.test import SimpleTestCase from django.urls import resolve, reverse diff --git a/accounts/views.py b/accounts/views.py index 95828d73..f3af3880 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -30,7 +30,7 @@ UserCreateSerializerValidator, ) from core.utils import Utils -from datahub.models import UserOrganizationMap +from participant.models import UserOrganizationMap from utils import login_helper, string_functions from utils.jwt_services import http_request_mutation diff --git a/datahub/tests/test_api_builder.py b/datahub/tests/test_api_builder.py index 13b7b39c..02a2a22f 100644 --- a/datahub/tests/test_api_builder.py +++ b/datahub/tests/test_api_builder.py @@ -1,7 +1,7 @@ from rest_framework.reverse import reverse from django.test import Client, TestCase import json -from datahub.models import DatasetV2, Organization, UserOrganizationMap +from participant.models import DatasetV2, Organization, UserOrganizationMap from accounts.models import User, UserRole from participant.tests.test_util import TestUtils from _pytest.monkeypatch import MonkeyPatch diff --git a/datahub/tests/test_datahub_views.py b/datahub/tests/test_datahub_views.py index c75d3678..b3062a73 100644 --- a/datahub/tests/test_datahub_views.py +++ b/datahub/tests/test_datahub_views.py @@ -8,7 +8,7 @@ from _pytest.monkeypatch import MonkeyPatch from django.db import models from accounts.models import User, UserManager, UserRole -from datahub.models import Datasets, Organization, UserOrganizationMap +from participant.models import Datasets, Organization, UserOrganizationMap from datahub.views import ParticipantViewSet from django.test import Client, TestCase from django.test.client import encode_multipart diff --git a/microsite/tests/test_home_view.py b/microsite/tests/test_home_view.py index 3b4d9c39..72b8ac56 100644 --- a/microsite/tests/test_home_view.py +++ b/microsite/tests/test_home_view.py @@ -2,7 +2,7 @@ from django.test import Client, TestCase from rest_framework import status import json -from datahub.models import Organization, UserOrganizationMap +from participant.models import Organization, UserOrganizationMap from accounts.models import User, UserRole org_for_participant = { From 66005104f1f8e5b1ece5af2cd619d2d5e13e4d58 Mon Sep 17 00:00:00 2001 From: Jai Date: Tue, 5 Mar 2024 17:29:39 +0530 Subject: [PATCH 4/5] pk api handling --- datasets/urls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasets/urls.py b/datasets/urls.py index 65bc0f84..a0d76a67 100644 --- a/datasets/urls.py +++ b/datasets/urls.py @@ -13,4 +13,5 @@ urlpatterns = [ path("v2/", include(router.urls)), + path("v2//", include(router.urls)), ] \ No newline at end of file From e0e866ca1a1ef8210e4fcd01e4ab040cabd7b21e Mon Sep 17 00:00:00 2001 From: Jai Date: Tue, 5 Mar 2024 17:30:25 +0530 Subject: [PATCH 5/5] pk api handling --- datasets/urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasets/urls.py b/datasets/urls.py index a0d76a67..65bc0f84 100644 --- a/datasets/urls.py +++ b/datasets/urls.py @@ -13,5 +13,4 @@ urlpatterns = [ path("v2/", include(router.urls)), - path("v2//", include(router.urls)), ] \ No newline at end of file