Tracking pull request to merge release-1.42.0 to master (#1091)

* update to 1.42.0

* update workflow and pipeline to use 1.42.0

* feat: allow editable comments for model year report discussion (#1093)

* feat: allow editable comments for model year report discussion

* chore: cleanup and rearrange file structure

* chore: linting

* feat: zeva-267 - edit credit application comments (#1094)

* feat: zeva-267 - edit credit application comments

* small change

* remove immer

Co-authored-by: Alex Zorkin <47334977+AlexZorkin@users.noreply.github.com>

* run backend test (#1099)

* comment out test

* fix: added filter that removes inactive users from email subscribed list (#1095)

* fix: zeva-445 - highlight new models on submit (#1098)

* fix: adds directorAction into condition for displaying 'return to analyst' button (#1101)

* fix: version fix for enums (#1107)

* fix: Zeva 443 - 21 error on old submissions (#1106)

* fix: adds exclude to has_been_awarded filter to remove any records that have a later timestamp than the current submission

* chore: linting

* fix: adds more to raw sql query for 11 errors so that icbc data uploaded after the sales submission would be excluded when calculating warnings (ie it will show appropriate 11 errors)

* Feat: Supplementary index links - ZEVA-444 (#1103)

* fix: reworked index link logic on model year report table

* chore: logic cleanup, linting

* use python 3.9 (#1113)

* add dev deployment

* update workflow depencency

* update apiVersion

* remove sschema spy

* chore: adds vehicleListSerializer so that vehicle list is returned faster, uses serializer method fields to return just necessary data rather than calling other serializers that return data that is not needed (#1112)

* feat: adds condition for displaying checkboxes and button bars on model year report pages (#1109)

* Fix: Model Year Report Tests for change log sales - ZEVA 432 (#1118)

* fix: enum fixes on credit test

* feat: model year reports testing harness fixed, assessment patch logic tested

* chore: pretty ignore file

* fix: preinitialize table render for validated-details view - ZEVA 460 (#1121)

* fix: preinitialize table render for validated-details view

* fix: logic fix to prevent re-rendering

* Update frontend base image (#1117)

* split dev deploy (#1127)

* Frontend try 1.42.0 (#1130)

Update the assemble file

* update dev deploy name

* feat: zeva-267 - add ability to delete comments (#1136)

* remove dev deployment

* -changes how action bar and signoff is conditionally rendered, so it now does not render if the report has been submitted, assessed, or reassessed (#1140)

Co-authored-by: Alex Zorkin <47334977+AlexZorkin@users.noreply.github.com>
Co-authored-by: tim738745 <98717409+tim738745@users.noreply.github.com>
Co-authored-by: Emily <44536222+emi-hi@users.noreply.github.com>

.github/workflows/build-new-python-image.yaml

@@ -0,0 +1,37 @@
+## For each release, the value of workflow name, branches, PR_NUMBER and RELEASE_NAME need to be adjusted accordingly
+## Also change the .pipelin/lib/config.js version number
+name: new-python-image
+
+on:
+  push:
+    branches: [ new-python39-image-1.42.0 ]
+  workflow_dispatch:
+  workflow_call:
+
+env:
+  ## The pull request number of the Tracking pull request to merge the release branch to main
+  PR_NUMBER: 1102
+  RELEASE_NAME: new-python39-image-1.42.0
+
+jobs:
+
+  ## This is the CI job
+  build:
+
+    name: Build ZEVA on Openshift
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+
+      ## it will checkout to /home/runner/work/zeva/zeva
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      # open it when zeva updated the python packages
+      - name: Run django tests 
+        uses: kuanfandevops/django-test-action@zeva-django-test
+        with:
+          settings-dir-path: "backend/zeva"
+          requirements-file: "backend/requirements.txt"
+          managepy-dir: backend

.github/workflows/build-release.yaml

@@ -1,17 +1,17 @@
 ## For each release, the value of workflow name, branches, PR_NUMBER and RELEASE_NAME need to be adjusted accordingly
 ## Also change the .pipelin/lib/config.js version number
-name: CI/CD ZEVA release-1.41.0
+name: ZEVA v1.42.0
 
 on:
   push:
-    branches: [ release-1.41.0 ]
+    branches: [ release-1.42.0 ]
   workflow_dispatch:
   workflow_call:
 
 env:
   ## The pull request number of the Tracking pull request to merge the release branch to main
-  PR_NUMBER: 1072
-  RELEASE_NAME: release-1.41.0
+  PR_NUMBER: 1091
+  RELEASE_NAME: release-1.42.0
 
 jobs:
 
@@ -28,6 +28,14 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v2
 
+      # open it when zeva updated the python packages
+      #- name: Run django tests 
+      #  uses: kuanfandevops/django-test-action@zeva-django-test
+      #  with:
+      #    settings-dir-path: "backend/zeva"
+      #    requirements-file: "backend/requirements.txt"
+      #    managepy-dir: backend
+
       ## Log in to Openshift with a token of service account
       - name: Log in to Openshift
         uses: redhat-actions/oc-login@v1
@@ -44,12 +52,39 @@ jobs:
           npm install
           npm run build -- --pr=${{ env.PR_NUMBER }} --env=build 
 
+  deploy-on-dev:
+
+    name: Deploy ZEVA on Dev Environment
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: build
+
+    steps:
+
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Log in to Openshift
+        uses: redhat-actions/oc-login@v1
+        with:
+          openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }}
+          openshift_token: ${{ secrets.OPENSHIFT_TOKEN }}
+          insecure_skip_tls_verify: true
+          namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools    
+
+      #- name: Run BCDK deployment on ZEVA Dev environment
+      #  run: |
+      #    cd .pipeline
+      #    echo "Deploying ZEVA ${{ env.RELEASE_NAME }} on Dev"
+      #    npm install
+      #    npm run deploy -- --pr=${{ env.PR_NUMBER }} --env=dev
+
   deploy-on-test:
 
     name: Deploy ZEVA on Test Environment
     runs-on: ubuntu-latest
     timeout-minutes: 60
-    needs: build
+    needs: deploy-on-dev
 
     steps:
 
@@ -112,4 +147,4 @@ jobs:
           cd .pipeline
           echo "Deploying ZEVA ${{ env.RELEASE_NAME }} on Prod"
           npm install
-          npm run deploy -- --pr=${{ env.PR_NUMBER }} --env=prod          
+          npm run deploy -- --pr=${{ env.PR_NUMBER }} --env=prod          

.jenkins/.pipeline/lib/config.js

@@ -1,7 +1,7 @@
 'use strict';
 const options= require('@bcgov/pipeline-cli').Util.parseArguments()
 const changeId = options.pr //aka pull-request
-const version = '1.0.0'
+const version = '1.42.0'
 const name = 'jenkins'
 const ocpName = 'apps.silver.devops'
 

.pipeline/lib/config.js

@@ -1,7 +1,7 @@
 'use strict';
 const options= require('@bcgov/pipeline-cli').Util.parseArguments()
 const changeId = options.pr //aka pull-request
-const version = '1.41.0'
+const version = '1.42.0'
 const name = 'zeva'
 const ocpName = 'apps.silver.devops'
 

.pipeline/lib/deploy.js

@@ -148,6 +148,7 @@ module.exports = settings => {
   })) 
 
   //deploy schemaspy
+  /*
   if(phase === 'dev') {
     objects = objects.concat(oc.processDeploymentTemplate(`${templatesLocalBaseUrl}/templates/schemaspy/schemaspy-dc.yaml`, {
       'param': {
@@ -161,7 +162,7 @@ module.exports = settings => {
         'OCP_NAME': phases[phase].ocpName
       }
     }))
-  }
+  }*/
 
   /** 
   //deploy rabbitmq, use docker image directly

backend/.s2i/bin/assemble

@@ -8,58 +8,104 @@ function should_collectstatic() {
   is_django_installed && [[ -z "$DISABLE_COLLECTSTATIC" ]]
 }
 
-# Install pipenv to the separate virtualenv to isolate it
+function virtualenv_bin() {
+    # New versions of Python (>3.6) should use venv module
+    # from stdlib instead of virtualenv package
+    python3.9 -m venv $1
+}
+
+# Install pipenv or micropipenv to the separate virtualenv to isolate it
 # from system Python packages and packages in the main
 # virtualenv. Executable is simlinked into ~/.local/bin
 # to be accessible. This approach is inspired by pipsi
 # (pip script installer).
-function install_pipenv() {
-  echo "---> Installing pipenv packaging tool ..."
-  VENV_DIR=$HOME/.local/venvs/pipenv
-  virtualenv $VENV_DIR
-  $VENV_DIR/bin/pip --isolated install -U pipenv
+function install_tool() {
+  echo "---> Installing $1 packaging tool ..."
+  VENV_DIR=$HOME/.local/venvs/$1
+  virtualenv_bin "$VENV_DIR"
+  # First, try to install the tool without --isolated which means that if you
+  # have your own PyPI mirror, it will take it from there. If this try fails, try it
+  # again with --isolated which ignores external pip settings (env vars, config file)
+  # and installs the tool from PyPI (needs internet connetion).
+  # $1$2 combines package name with [extras] or version specifier if is defined as $2```
+  if ! $VENV_DIR/bin/pip install -U $1$2; then
+    echo "WARNING: Installation of $1 failed, trying again from official PyPI with pip --isolated install"
+    $VENV_DIR/bin/pip install --isolated -U $1$2  # Combines package name with [extras] or version specifier if is defined as $2```
+  fi
   mkdir -p $HOME/.local/bin
-  ln -s $VENV_DIR/bin/pipenv $HOME/.local/bin/pipenv
+  ln -s $VENV_DIR/bin/$1 $HOME/.local/bin/$1
 }
 
 set -e
 
+# First of all, check that we don't have disallowed combination of ENVs
+if [[ ! -z "$ENABLE_PIPENV" && ! -z "$ENABLE_MICROPIPENV" ]]; then
+  echo "ERROR: Pipenv and micropipenv cannot be enabled at the same time!"
+  # podman/buildah does not relay this exit code but it will be fixed hopefuly
+  # https://github.com/containers/buildah/issues/2305
+  exit 3
+fi
+
 shopt -s dotglob
 echo "---> Installing application source ..."
-mv /tmp/src/* ./
+mv /tmp/src/* "$HOME"
 
-if [[ ! -z "$UPGRADE_PIP_TO_LATEST" || ! -z "$ENABLE_PIPENV" ]]; then
-  echo "---> Upgrading pip to latest version ..."
-  pip install -U pip setuptools wheel
+# set permissions for any installed artifacts
+fix-permissions /opt/app-root -P
+
+
+if [[ ! -z "$UPGRADE_PIP_TO_LATEST" ]]; then
+  echo "---> Upgrading pip, setuptools and wheel to latest version ..."
+  if ! pip install -U pip setuptools wheel; then
+    echo "WARNING: Installation of the latest pip, setuptools and wheel failed, trying again from official PyPI with pip --isolated install"
+    pip install --isolated -U pip setuptools wheel
+  fi
 fi
 
 if [[ ! -z "$ENABLE_PIPENV" ]]; then
-  install_pipenv
+  if [[ ! -z "$PIN_PIPENV_VERSION" ]]; then
+    # Add == as a prefix to pipenv version, if defined
+    PIN_PIPENV_VERSION="==$PIN_PIPENV_VERSION"
+  fi
+  install_tool "pipenv" "$PIN_PIPENV_VERSION"
   echo "---> Installing dependencies via pipenv ..."
   if [[ -f Pipfile ]]; then
     pipenv install --deploy
   elif [[ -f requirements.txt ]]; then
     pipenv install -r requirements.txt
   fi
-  pipenv check
+  # pipenv check
+elif [[ ! -z "$ENABLE_MICROPIPENV" ]]; then
+  install_tool "micropipenv" "[toml]"
+  echo "---> Installing dependencies via micropipenv ..."
+  # micropipenv detects Pipfile.lock and requirements.txt in this order
+  micropipenv install --deploy 
 elif [[ -f requirements.txt ]]; then
-  echo "---> Installing dependencies ..."
-  pip install -i https://$ARTIFACTORY_USER:$ARTIFACTORY_PASSWORD@artifacts.developer.gov.bc.ca/artifactory/api/pypi/pypi-remote/simple --upgrade pip
-  pip install -i https://$ARTIFACTORY_USER:$ARTIFACTORY_PASSWORD@artifacts.developer.gov.bc.ca/artifactory/api/pypi/pypi-remote/simple -r requirements.txt
-elif [[ -f setup.py ]]; then
-  echo "---> Installing application ..."
-  python setup.py develop
+  if [[ -z "${ARTIFACTORY_USER}" ]]; then
+    echo "---> Installing dependencies from external repo ..."
+    pip install -r requirements.txt
+  else
+    echo "---> Installing dependencies from artifactory ..."
+    pip install -i https://$ARTIFACTORY_USER:$ARTIFACTORY_PASSWORD@artifacts.developer.gov.bc.ca/artifactory/api/pypi/pypi-remote/simple -r requirements.txt
+  fi
 fi
 
-
+if [[ -f setup.py && -z "$DISABLE_SETUP_PY_PROCESSING" ]]; then
+  echo "---> Installing application ..."
+  pip install .
+fi
 
 if should_collectstatic; then
   (
     echo "---> Collecting Django static files ..."
 
-    APP_HOME=${APP_HOME:-.}
-    # Look for 'manage.py' in the directory specified by APP_HOME, or the current directory
-    manage_file=$APP_HOME/manage.py
+    APP_HOME=$(readlink -f "${APP_HOME:-.}")
+    # Change the working directory to APP_HOME
+    PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+    cd "$APP_HOME"
+
+    # Look for 'manage.py' in the current directory
+    manage_file=./manage.py
 
     if [[ ! -f "$manage_file" ]]; then
       echo "WARNING: seems that you're using Django, but we could not find a 'manage.py' file."
@@ -75,20 +121,8 @@ if should_collectstatic; then
     fi
 
     python $manage_file collectstatic --noinput
-
   )
 fi
 
-echo "---> current folder is "
-pwd
-
-# Run unit tests in build stage with the code below
-# echo "--> running Django unit tests"
-
-# python $manage_file test
-
 # set permissions for any installed artifacts
-fix-permissions /opt/app-root
-echo "---> current folder2 is "
-pwd
-ls -lrt
+fix-permissions /opt/app-root -P

backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV PYTHONUNBUFFERED=1
 

backend/api/models/sales_submission_content.py

@@ -84,7 +84,10 @@ def vehicle(self):
 
     @property
     def icbc_verification(self):
-        q = 'select * from icbc_registration_data where vin=\'{}\' limit 1'.format(self.xls_vin)
+        q = 'select * from icbc_registration_data join icbc_upload_date on \
+            icbc_upload_date.id = icbc_upload_date_id where \
+                vin=\'{}\' and upload_date < \'{}\' limit 1'.format(
+                    self.xls_vin, self.update_timestamp)
         registration = IcbcRegistrationData.objects.raw(q)
         if registration:
             return registration[0]
@@ -113,6 +116,8 @@ def is_already_awarded(self):
             submission_id=self.submission_id
         ).filter(
             vin=self.xls_vin
+        ).exclude(
+            create_timestamp__gte=self.update_timestamp
         ).first()
 
         if has_been_awarded:

backend/api/serializers/sales_submission_comment.py

@@ -22,10 +22,15 @@ def get_create_user(self, obj):
         serializer = MemberSerializer(user, read_only=True)
         return serializer.data
 
+    def update(self, instance, validated_data):
+        instance.comment = validated_data.get("comment")
+        instance.save()
+        return instance
+
     class Meta:
         model = SalesSubmissionComment
         fields = (
-            'id', 'comment', 'create_timestamp', 'create_user','to_govt'
+            'id', 'comment', 'create_timestamp', 'create_user','to_govt', 'update_timestamp'
         )
         read_only_fields = (
             'id',