Merge pull request #2168 from City-of-Helsinki/develop

Merge develop to main (2023-08-09)

.env.benefit-backend.example

@@ -61,14 +61,18 @@ MEDIA_ROOT=/app/var/media
 CSRF_COOKIE_NAME=yjdhcsrftoken
 CSRF_TRUSTED_ORIGINS="localhost:3000,localhost:3100"
 
-YRTTI_TIMEOUT=30
-YRTTI_BASIC_INFO_PATH=https://yrtti-integration-test.agw.arodevtest.hel.fi/api/BasicInfo
-SERVICE_BUS_INFO_PATH=https://ytj-integration-test.agw.arodevtest.hel.fi/api/GetCompany
+YRTTI_BASE_URL=https://yrtti-integration-test.agw.arodevtest.hel.fi/api
 YRTTI_AUTH_PASSWORD=
 YRTTI_AUTH_USERNAME=helsinkilisatest
+YRTTI_TIMEOUT=30
+YRTTI_SEARCH_LIMIT=10
+YRTTI_DISABLE=0
 
+SERVICE_BUS_BASE_URL=https://ytj-integration-test.agw.arodevtest.hel.fi/api
 SERVICE_BUS_AUTH_PASSWORD=
 SERVICE_BUS_AUTH_USERNAME=helsinkilisatest
+SERVICE_BUS_TIMEOUT=30
+SERVICE_BUS_SEARCH_LIMIT=10
 
 SEND_AUDIT_LOG=0
 
@@ -98,10 +102,3 @@ SENTRY_ENVIRONMENT=local
 # for Mailhog inbox
 EMAIL_HOST=mailhog
 EMAIL_PORT=1025
-
-# Variables for using a S3 compatible disk in local development environment in upcoming staging / production environments
-USE_S3=1
-S3_ENDPOINT_URL="http://minio:9000"
-S3_ACCESS_KEY_ID=minio-root
-S3_SECRET_ACCESS_KEY=minio-pass
-S3_STORAGE_BUCKET_NAME=local-s3-bucket

azure-pipelines/helsinkilisa-review.yml

@@ -0,0 +1,65 @@
+#
+# Review pipeline. Run build and deploy for Platta test environments.
+# Pipeline runs different tests e.g. unittest and browser tests.
+#
+# Continuous integration (CI) triggers cause a pipeline to run whenever you push
+# an update to the specified branches or you push specified tags.
+# only PR trigger pipeline
+trigger: none
+
+# Pull request (PR) triggers cause a pipeline to run whenever a pull request is
+# opened with one of the specified target branches, or when updates are made to 
+# such a pull request.
+#
+# GitHub creates a new ref when a pull request is created. The ref points to a 
+# merge commit, which is the merged code between the source and target branches 
+# of the pull request.
+#
+# Opt out of pull request validation 
+pr:
+  # PR target branch
+  branches:
+    include:
+    - develop
+  paths:
+    include:
+      - azure-pipelines/helsinkilisa-review.yml
+      - backend/docker/benefit.Dockerfile
+      - backend/benefit/**
+      - backend/shared/**
+      - frontend/*
+      - frontend/benefit/**
+      - frontend/shared/**
+    exclude:
+      - README.md
+      - backend/kesaseteli/**
+      - backend/tet/**
+      - frontend/kesaseteli/**
+      - frontend/tet/**
+      - frontend/**/browser-tests
+      - frontend/**/__tests__
+
+# By default, use self-hosted agents
+pool: Default
+
+resources:
+  repositories:
+  # Azure DevOps repository
+  - repository: yjdh-helsinkilisa-pipelines
+    type: git
+    # Azure DevOps project/repository
+    name: yjdh-helsinkilisa/yjdh-helsinkilisa-pipelines
+
+extends:
+  # Filename in Azure DevOps Repository (note possible -ui or -api)
+  # Django example: azure-pipelines-PROJECTNAME-api-release.yml
+  # Drupal example: azure-pipelines-drupal-release.yml
+  template: azure-pipelines-helsinkilisa-review.yml@yjdh-helsinkilisa-pipelines
+    # Application build arguments and config map values as key value pairs.
+    # The values here will override the values defined in the yjdh-benefit-pipelines repository.
+    # for example
+#  parameters:
+#    buildArgs:
+#      NEXT_PUBLIC_DEBUG: 0
+#    configMap: # pod environment variables
+#      DEBUG: 0

backend/benefit/README.md

@@ -56,7 +56,6 @@ Set default permissions
 This creates permissions for the handler's group so they have access to the Terms in
 the django admin.
 
-
 ### Configure docker environment
 
 In the yjdh project root, set up the .env.benefit-backend file: `cp .env.benefit-backend.example .env.benefit-backend`
@@ -86,6 +85,7 @@ The project is now running at [localhost:8000](https://localhost:8000)
 ### Updating translations
 
 In `backend/benefit/`:
+
 - Run `python manage.py makemessages --no-location -l fi -l sv -l en`
 - Run `python manage.py compilemessages`
 
@@ -98,7 +98,7 @@ DUMMY_COMPANY_FORM_CODE can be set to test with different company_form parameter
 To seed the database with some mock application data, run `python manage.py seed`
 , which by default generates 10 applications for each of the seven possible application statuses and one attachment with a .pdf-file for each of them. To generate more applications, use the optional `--number` flag, for example, running `python manage.py seed --number=30` creates 30 applications of each status. **Note that running the command deletes all previous application data from the database and clears the media folder.**
 
-[Mailhog](https://github.com/mailhog) is available for the local development environment (localhost:8025)[http://localhost:8025/] for previewing 
+[Mailhog](https://github.com/mailhog) is available for the local development environment (localhost:8025)[http://localhost:8025/] for previewing
 and testing the emails sent by the application after setting the `EMAIL_HOST` and `EMAIL_PORT` as in the `.env.benefit-backend.example`.
 
 **Using LOAD_FIXTURES=1 is recommended for local testing** as it loads e.g. default
@@ -176,7 +176,7 @@ and redoc documentation at [https://localhost:8000/api_docs/redoc/](https://loca
 
 ## Scheduled jobs
 
-Jobs can be scheduled using the Django extensions-package and setting the jobs to run as a cronjob. 
+Jobs can be scheduled using the Django extensions-package and setting the jobs to run as a cronjob.
 Currently configured jobs (registered in the `applications/jobs`-directory):
 
 - Daily: check applications that have been in the cancelled state for 30 or more days and delete them.
@@ -208,24 +208,8 @@ env variables / settings are provided by Azure blob storage:
 - `AZURE_ACCOUNT_KEY`
 - `AZURE_CONTAINER`
 
-An AWS S3 compatible disk storage can be configured with the following environment variables.
-
-- `USE_S3`
-- `S3_ENDPOINT_URL`
-- `S3_ACCESS_KEY_ID`
-- `S3_SECRET_ACCESS_KEY`
-- `S3_STORAGE_BUCKET_NAME`
-
-[MinIO](https://min.io/) can been configured to work as the AWS S3 compatible file storage on the local development environment. The MinIO admin panel can be accessed at (localhost:9090)[http://localhost:9090/].
-See `.env.benefit-backend.example` for the  Minio variables and credentials.
-
-**Note**
-As tests freeze time with [freezegun](https://github.com/spulec/freezegun), the MinIO requests fail when running tests with exception `botocore.exceptions.ClientError: An error occurred (RequestTimeTooSkewed) when calling the PutObject operation: The difference between the request time and the server's time is too large.`
-
-Switch from MinIO to the local disk by setting `USE_S3=0` before running the pytest tests. For now, the only workaround to run tests with S3 enabled is to set host machine date to the date that is used in tests: `2021-06-04 00:00:00 (UTC)`.
-
-
 ## Sentry error monitoring
-The `local`, `development` and `testing` environments are connected to the Sentry instance at [`https://sentry.test.hel.ninja/`](https://sentry.test.hel.ninja/) under the `yjdh-benefit`-team. 
-There are separate Sentry projects for the Django api (`yjdh-benefit-api`), handler UI (`yjdh-benefit-handler`) and applicant UI (`yjdh-benefit-applicant`). 
+
+The `local`, `development` and `testing` environments are connected to the Sentry instance at [`https://sentry.test.hel.ninja/`](https://sentry.test.hel.ninja/) under the `yjdh-benefit`-team.
+There are separate Sentry projects for the Django api (`yjdh-benefit-api`), handler UI (`yjdh-benefit-handler`) and applicant UI (`yjdh-benefit-applicant`).
 To limit the amount of possibly sensitive data sent to Sentry, the same configuration as in kesaseteli is used by default, see [`https://github.com/City-of-Helsinki/yjdh/pull/779`](https://github.com/City-of-Helsinki/yjdh/pull/779).

backend/benefit/applications/admin.py

@@ -45,10 +45,11 @@ class ApplicationAdmin(admin.ModelAdmin):
         AttachmentInline,
         CalculationInline,
     )
-    list_filter = ("status", "company")
+    list_filter = ("status", "application_origin", "company")
     list_display = (
         "id",
         "status",
+        "application_origin",
         "application_number",
         "company_name",
         "company_contact_person_email",

backend/benefit/applications/api/v1/application_batch_views.py

@@ -191,7 +191,7 @@ def assign_applications(self, request):
         def create_application_batch_by_ids(app_status, apps):
             if apps:
                 batch = ApplicationBatch.objects.create(
-                    proposal_for_decision=app_status
+                    proposal_for_decision=app_status, handler=request.user
                 )
                 return batch
 
@@ -209,16 +209,23 @@ def create_application_batch_by_ids(app_status, apps):
         )
 
         # Try finding an existing batch
-        batch = (
-            ApplicationBatch.objects.filter(
-                status=ApplicationBatchStatus.DRAFT, proposal_for_decision=app_status
-            ).first()
-        ) or create_application_batch_by_ids(
-            app_status,
-            apps,
-        )
+        try:
+            batch = (
+                ApplicationBatch.objects.filter(
+                    status=ApplicationBatchStatus.DRAFT,
+                    proposal_for_decision=app_status,
+                ).first()
+            ) or create_application_batch_by_ids(
+                app_status,
+                apps,
+            )
+        except BatchTooManyDraftsError:
+            return Response(
+                {"errorKey": "batchInvalidDraftAlreadyExists"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
 
-        if batch:
+        if batch and batch.status == ApplicationBatchStatus.DRAFT:
             apps.update(batch=batch)
             batch = ApplicationBatchSerializer(batch)
             return Response(batch.data, status=status.HTTP_200_OK)
@@ -239,16 +246,22 @@ def deassign_applications(self, request, pk=None):
         application_ids = request.data.get("application_ids")
         batch = self.get_batch(pk)
 
-        apps = Application.objects.filter(
+        deassign_apps = Application.objects.filter(
+            batch=batch,
             pk__in=application_ids,
             status__in=[ApplicationStatus.ACCEPTED, ApplicationStatus.REJECTED],
-            batch=batch,
         )
-        if apps:
-            for app in apps:
+        if deassign_apps:
+            for app in deassign_apps:
                 app.batch = None
                 app.save()
-            return Response(status=status.HTTP_200_OK)
+            remaining_apps = Application.objects.filter(batch=batch)
+            if len(remaining_apps) == 0:
+                batch.delete()
+            return Response(
+                {"remainingApps": len(remaining_apps)}, status=status.HTTP_200_OK
+            )
+
         return Response(
             {"detail": "Applications were not applicable to be detached."},
             status=status.HTTP_404_NOT_FOUND,
@@ -264,23 +277,29 @@ def status(self, request, pk=None):
         batch = self.get_batch(pk)
         if new_status not in [
             ApplicationBatchStatus.DRAFT,
+            ApplicationBatchStatus.AHJO_REPORT_CREATED,
             ApplicationBatchStatus.AWAITING_AHJO_DECISION,
             ApplicationBatchStatus.DECIDED_ACCEPTED,
             ApplicationBatchStatus.DECIDED_REJECTED,
+            ApplicationBatchStatus.SENT_TO_TALPA,
         ]:
             return Response(status=status.HTTP_400_BAD_REQUEST)
 
         if new_status in [
             ApplicationBatchStatus.DECIDED_ACCEPTED,
             ApplicationBatchStatus.DECIDED_REJECTED,
         ]:
-            # Archive all applications if this batch will be completed
-            Application.objects.filter(batch=batch).update(archived=True)
-
-            # Patch all required fields for batch completion
+            # Patch all required fields after batch inspection
             for key in request.data:
                 setattr(batch, key, request.data.get(key))
 
+        if new_status in [
+            ApplicationBatchStatus.SENT_TO_TALPA,
+        ]:
+            # Archive all applications if this batch will be completed
+            Application.objects.filter(batch=batch).update(archived=True)
+
+        previous_status = batch.status
         batch.status = new_status
 
         try:
@@ -309,6 +328,7 @@ def status(self, request, pk=None):
             {
                 "id": batch.id,
                 "status": batch.status,
+                "previousStatus": previous_status,
                 "decision": batch.proposal_for_decision,
             },
             status=status.HTTP_200_OK,