From 51834bac02b1d976bf4de149c213aac1629a3a28 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 3 Aug 2021 09:46:10 +0100 Subject: [PATCH 1/9] feat: add `not_opened` field to submission model see: nditech/apollo-issues#108 --- apollo/submissions/models.py | 1 + .../b3ccd538959e_add_not_opened_field.py | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 migrations/versions/b3ccd538959e_add_not_opened_field.py diff --git a/apollo/submissions/models.py b/apollo/submissions/models.py index a35353fbf..eae61ea03 100644 --- a/apollo/submissions/models.py +++ b/apollo/submissions/models.py @@ -125,6 +125,7 @@ class Submission(BaseModel): passive_deletes=True)) conflicts = db.Column(JSONB) unreachable = db.Column(db.Boolean, default=False, nullable=False) + not_opened = db.Column(db.Boolean, default=False, nullable=False) geom = db.Column(Geometry('POINT', srid=4326)) verified_fields = db.Column(JSONB, default=[]) diff --git a/migrations/versions/b3ccd538959e_add_not_opened_field.py b/migrations/versions/b3ccd538959e_add_not_opened_field.py new file mode 100644 index 000000000..abda1be8b --- /dev/null +++ b/migrations/versions/b3ccd538959e_add_not_opened_field.py @@ -0,0 +1,32 @@ +"""add not_opened field + +Revision ID: b3ccd538959e +Revises: e00e521a0434 +Create Date: 2021-08-03 09:41:18.244493 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'b3ccd538959e' +down_revision = 'e00e521a0434' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + 'submission', + sa.Column('not_opened', sa.Boolean(), nullable=False, + server_default='False')) + op.alter_column('submission', 'not_opened', server_default=None) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('submission', 'not_opened') + # ### end Alembic commands ### From 76f76ef450da1626fd240ecfb528b10c5fc834dd Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 3 Aug 2021 09:47:43 +0100 Subject: [PATCH 2/9] chore: remove unused import and reorder imports --- migrations/versions/b3ccd538959e_add_not_opened_field.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/migrations/versions/b3ccd538959e_add_not_opened_field.py b/migrations/versions/b3ccd538959e_add_not_opened_field.py index abda1be8b..f6dab8018 100644 --- a/migrations/versions/b3ccd538959e_add_not_opened_field.py +++ b/migrations/versions/b3ccd538959e_add_not_opened_field.py @@ -5,9 +5,8 @@ Create Date: 2021-08-03 09:41:18.244493 """ -from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import postgresql +from alembic import op # revision identifiers, used by Alembic. revision = 'b3ccd538959e' From 8ebfe0d46c0956403bd7e8e22eef5ffa0b5dfdad Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 3 Aug 2021 10:02:18 +0100 Subject: [PATCH 3/9] feat: implement 'opened' editing, filters & UI --- .../macros/quality_assurance_list_filter.html | 3 ++ .../macros/submission_list_filter.html | 4 +++ .../templates/frontend/submission_edit.html | 19 ++++++++++++ apollo/submissions/filters.py | 31 +++++++++++++++++++ apollo/submissions/forms.py | 1 + apollo/submissions/views_submissions.py | 6 ++++ 6 files changed, 64 insertions(+) diff --git a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html index fbe2c2e91..5df55aac4 100644 --- a/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/quality_assurance_list_filter.html @@ -44,6 +44,9 @@
{{ filter_form.participant_role(class_='form-control custom-select') }} +
+ + {{ filter_form.station_status(class_='form-control custom-select') }}
{%- if form.show_moment %}
diff --git a/apollo/frontend/templates/frontend/macros/submission_list_filter.html b/apollo/frontend/templates/frontend/macros/submission_list_filter.html index fd6d8e23d..c75d7558f 100644 --- a/apollo/frontend/templates/frontend/macros/submission_list_filter.html +++ b/apollo/frontend/templates/frontend/macros/submission_list_filter.html @@ -70,6 +70,10 @@ {{ filter_form.participant_role(class_='form-control custom-select') }}
+
+ + {{ filter_form.station_status(class_='form-control custom-select') }} +
{%- if form.show_moment %}
diff --git a/apollo/frontend/templates/frontend/submission_edit.html b/apollo/frontend/templates/frontend/submission_edit.html index 3116b642e..4070e8e29 100644 --- a/apollo/frontend/templates/frontend/submission_edit.html +++ b/apollo/frontend/templates/frontend/submission_edit.html @@ -243,6 +243,25 @@
{{ _('Data Validation Errors') }}
  {%- endif %} + + {{ _('Was station opened?') }} + + + {%- if readonly %} + {{ submission_form.not_opened(**{'class_': 'tracked', 'data-toggle': 'switchbutton', 'data-width': '80', 'data-onlabel': _('Not Opened'), 'data-onstyle': 'danger', 'data-offlabel': _('Opened'), 'data-offstyle': 'primary', 'data-size': 'sm', 'disabled': 'disabled'}) }} + {%- else %} + {{ submission_form.not_opened(**{'class_': 'tracked', 'data-toggle': 'switchbutton', 'data-width': '80', 'data-onlabel': _('Not Opened'), 'data-onstyle': 'danger', 'data-offlabel': _('Opened'), 'data-offstyle': 'primary', 'data-size': 'sm'}) }} + {%- endif %} + + {%- if form.form_type == 'CHECKLIST' and not form.untrack_data_conflicts -%} + {%- for sibling_form in sibling_forms %} + +   + + {% endfor %} +   + {%- endif %} + {% if perms.edit_submission_quarantine_status.can() %} {{ _('Quarantine Status') }} diff --git a/apollo/submissions/filters.py b/apollo/submissions/filters.py index 81896e82f..0fe9b2f17 100644 --- a/apollo/submissions/filters.py +++ b/apollo/submissions/filters.py @@ -376,6 +376,22 @@ def filter(self, query, value, **kwargs): return (None, None) +class StationStatusFilter(ChoiceFilter): + def filter(self, query, value, **kwargs): + if value and value == '1': + return ( + models.Submission.not_opened == True, # noqa + None + ) + elif value: + return ( + models.Submission.not_opened == False, # noqa + None + ) + + return (None, None) + + class DateFilter(CharFilter): def filter(self, query, value, **kwargs): if value: @@ -744,6 +760,13 @@ def make_submission_list_filter(event, form, filter_on_locations=False): ('1', _('No Signal')) ) ) + attributes['station_status'] = StationStatusFilter( + choices=( + ('', _('Station Status')), + ('0', _('Opened')), + ('1', _('Not opened')) + ) + ) attributes['date'] = DateFilter() attributes['fsn'] = FormSerialNumberFilter() attributes['participant_role'] = make_participant_role_filter( @@ -799,6 +822,14 @@ class QualityAssuranceConditionsForm(Form): ) ) + attributes['station_status'] = StationStatusFilter( + choices=( + ('', _('Station Status')), + ('0', _('Open')), + ('1', _('Not opened')) + ) + ) + # participant id and location attributes['participant_id'] = ParticipantIDFilter() attributes['location'] = AJAXLocationFilter() diff --git a/apollo/submissions/forms.py b/apollo/submissions/forms.py index 0c6f28aa5..2a9b8ca8a 100644 --- a/apollo/submissions/forms.py +++ b/apollo/submissions/forms.py @@ -203,6 +203,7 @@ def make_submission_edit_form_class(event, form): form_fields['validate'] = validate_location form_fields['unreachable'] = fields.BooleanField() + form_fields['not_opened'] = fields.BooleanField() return type( 'SubmissionEditForm', diff --git a/apollo/submissions/views_submissions.py b/apollo/submissions/views_submissions.py index 311b95930..1cc4d1f38 100644 --- a/apollo/submissions/views_submissions.py +++ b/apollo/submissions/views_submissions.py @@ -660,6 +660,7 @@ def submission_edit(submission_id): initial_data.update(location=submission.location_id) initial_data.update(participant=submission.participant_id) initial_data.update(unreachable=submission.unreachable) + initial_data.update(not_opened=submission.not_opened) if submission.quarantine_status: initial_data.update( quarantine_status=submission.quarantine_status.code) @@ -706,6 +707,7 @@ def submission_edit(submission_id): for sibling in sibling_submissions: initial_data = sibling.data initial_data.update(unreachable=sibling.unreachable) + initial_data.update(not_opened=sibling.not_opened) if sibling.quarantine_status: initial_data.update( quarantine_status=sibling.quarantine_status.code) @@ -1015,6 +1017,7 @@ def submission_edit(submission_id): new_quarantine_status = submission_form.data.get( 'quarantine_status') new_offline_status = submission_form.unreachable.data + new_station_status = submission_form.not_opened.data if permissions.edit_submission_verification_status.can(): new_verified_fields = \ @@ -1050,6 +1053,9 @@ def submission_edit(submission_id): if new_offline_status != submission.unreachable: changed = True update_params['unreachable'] = new_offline_status + if new_station_status != submission.not_opened: + changed = True + update_params['not_opened'] = new_station_status changed_fields = [] From 6409c72b24174348e898c6c9b037766a8604fe06 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 3 Aug 2021 12:47:46 +0100 Subject: [PATCH 4/9] feat: update master "opened" status --- apollo/submissions/api/views.py | 1 + apollo/submissions/models.py | 22 ++++++++++++++++++++++ apollo/submissions/views_submissions.py | 1 + 3 files changed, 24 insertions(+) diff --git a/apollo/submissions/api/views.py b/apollo/submissions/api/views.py index c500a454e..482e79dfc 100644 --- a/apollo/submissions/api/views.py +++ b/apollo/submissions/api/views.py @@ -342,6 +342,7 @@ def submission(): submission.update_related(data) if submission.master is not None: submission.update_master_offline_status() + submission.update_master_opened_status() update_submission_version(submission) message_text = make_message_text(form, participant, data) diff --git a/apollo/submissions/models.py b/apollo/submissions/models.py index eae61ea03..8b019cb2f 100644 --- a/apollo/submissions/models.py +++ b/apollo/submissions/models.py @@ -336,6 +336,28 @@ def update_master_offline_status(self): if master_offline_status != self.master.unreachable: db.session.add(self.master) + def update_master_opened_status(self): + if self.master is None: + return + + siblings = self.siblings + master_opened_status = self.master.not_opened + + if siblings: + if ( + all([s.not_opened for s in self.siblings]) and + self.not_opened + ): + self.master.not_opened = True + else: + self.master.not_opened = False + else: + self.master.not_opened = self.not_opened + + # if the offline status changed in any way + if master_opened_status != self.master.not_opened: + db.session.add(self.master) + def compute_conflict_tags(self, tags=None): # don't compute if the 'track conflicts' flag is not set # on the form diff --git a/apollo/submissions/views_submissions.py b/apollo/submissions/views_submissions.py index 1cc4d1f38..01a849728 100644 --- a/apollo/submissions/views_submissions.py +++ b/apollo/submissions/views_submissions.py @@ -1123,6 +1123,7 @@ def submission_edit(submission_id): submission.update_related(data) submission.update_master_offline_status() + submission.update_master_opened_status() db.session.add_all(attachments) for attachment in deleted_attachments: From ba9f756280047f04a9b15b87e4e07ed854a8f712 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 3 Aug 2021 12:48:26 +0100 Subject: [PATCH 5/9] feat: add computation of "not opened" on dashboard --- apollo/frontend/dashboard.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/apollo/frontend/dashboard.py b/apollo/frontend/dashboard.py index cd2f6c5c0..71d68e720 100644 --- a/apollo/frontend/dashboard.py +++ b/apollo/frontend/dashboard.py @@ -136,6 +136,20 @@ def _get_group_coverage(query, form, group, location_type): else: offline_query = query.filter(false()) + if group_tags: + not_opened_query = query.filter( + and_( + Submission.not_opened == True, # noqa + not_( + and_( + Submission.data.has_all(array(group_tags)), + Submission.not_opened == True + ) + ) + )) + else: + not_opened_query = query.filter(false()) + dataset = defaultdict(dict) for loc_id, loc_name, count in _get_coverage_results( @@ -178,6 +192,14 @@ def _get_group_coverage(query, form, group, location_type): 'name': loc_name }) + for loc_id, loc_name, count in _get_coverage_results( + not_opened_query, depth_info.depth): + dataset[loc_name].update({ + 'Closed': count, + 'id': loc_id, + 'name': loc_name + }) + for name in sorted(dataset.keys()): loc_data = dataset.get(name) loc_data.setdefault('Complete', 0) @@ -185,6 +207,7 @@ def _get_group_coverage(query, form, group, location_type): loc_data.setdefault('Missing', 0) loc_data.setdefault('Partial', 0) loc_data.setdefault('Offline', 0) + loc_data.setdefault('Closed', 0) coverage_list.append(loc_data) @@ -257,12 +280,27 @@ def _get_global_coverage(query, form): else: offline_query = query.filter(false()) + if group_tags: + not_opened_query = query.filter( + and_( + Submission.not_opened == True, # noqa + not_( + and_( + Submission.data.has_all(array(group_tags)), + Submission.not_opened == True + ) + ) + )) + else: + not_opened_query = query.filter(false()) + data = { 'Complete': complete_query.count(), 'Conflict': conflict_query.count(), 'Missing': missing_query.count(), 'Partial': partial_query.count(), 'Offline': offline_query.count(), + 'Closed': not_opened_query.count(), 'name': group['name'], 'slug': group['slug'] } From 3edba35d769ce38256209575f7b0a16a7b033fe0 Mon Sep 17 00:00:00 2001 From: Odumosu Dipo Date: Tue, 3 Aug 2021 12:50:58 +0100 Subject: [PATCH 6/9] feat: dashboard rendering for not opened --- .../frontend/templates/frontend/dashboard.html | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/apollo/frontend/templates/frontend/dashboard.html b/apollo/frontend/templates/frontend/dashboard.html index ce2849afb..127026c74 100644 --- a/apollo/frontend/templates/frontend/dashboard.html +++ b/apollo/frontend/templates/frontend/dashboard.html @@ -58,7 +58,7 @@
{% if not location and not request.args.location %} {%- if session['dashboard_chart_type'] == 'bar' %}