From a411986c5b293a39211292eb65891300e9073165 Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Tue, 24 Sep 2024 11:53:57 -0400 Subject: [PATCH 1/6] upgrading pyxform to 2.0.3 and fixing deployment errors due to format changes with upgrade --- dependencies/pip/dev_requirements.txt | 4 ++-- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 4 ++-- kobo/apps/openrosa/apps/viewer/models/data_dictionary.py | 8 +++++++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index acbd45e9ed..c48651a35d 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -409,7 +409,7 @@ oauthlib==3.2.2 # -r dependencies/pip/requirements.in # django-oauth-toolkit # requests-oauthlib -openpyxl==3.0.9 +openpyxl==3.1.2 # via # -r dependencies/pip/requirements.in # pyxform @@ -530,7 +530,7 @@ pytz==2024.1 # via # flower # pandas -pyxform==1.9.0 +pyxform==2.0.3 # via # -r dependencies/pip/requirements.in # formpack diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index e067f8bbb4..542f37120b 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -77,7 +77,7 @@ openpyxl psycopg pymongo python-dateutil -pyxform==1.9.0 +pyxform==2.0.3 requests regex responses diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 049bbb789f..9b38538fa8 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -335,7 +335,7 @@ oauthlib==3.2.2 # -r dependencies/pip/requirements.in # django-oauth-toolkit # requests-oauthlib -openpyxl==3.0.9 +openpyxl==3.1.2 # via # -r dependencies/pip/requirements.in # pyxform @@ -411,7 +411,7 @@ pytz==2024.1 # via # flower # pandas -pyxform==1.9.0 +pyxform==2.0.3 # via # -r dependencies/pip/requirements.in # formpack diff --git a/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py b/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py index 4048e82208..0c942d3f4e 100644 --- a/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py +++ b/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py @@ -1,4 +1,5 @@ # coding: utf-8 +import io import os import re from xml.dom import Node @@ -156,7 +157,12 @@ def add_instances(self): def save(self, *args, **kwargs): if self.xls: - survey = create_survey_from_xls(self.xls) + file_obj = self.xls.read() + # Wrap the file data in a BytesIO object to simulate a file + xls_bytes_io = io.BytesIO(file_obj) + xls_bytes_io.name = self.xls.name + + survey = create_survey_from_xls(xls_bytes_io) survey.update({ 'name': survey.id_string, }) From b82ad277475840a03707431f4ac8faa5d254eb3c Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Tue, 24 Sep 2024 14:14:59 -0400 Subject: [PATCH 2/6] update tests to align with pyxform upgrade which includes updating XML fixtures and test assertions to match the new format --- .../tests/fixtures/Transportation Form.xml | 326 +++++++---------- .../transportation/transportation.xml | 329 +++++++----------- kpi/tests/test_asset_snapshots.py | 2 +- 3 files changed, 262 insertions(+), 395 deletions(-) diff --git a/kobo/apps/openrosa/apps/api/tests/fixtures/Transportation Form.xml b/kobo/apps/openrosa/apps/api/tests/fixtures/Transportation Form.xml index 073131011f..98260af363 100644 --- a/kobo/apps/openrosa/apps/api/tests/fixtures/Transportation Form.xml +++ b/kobo/apps/openrosa/apps/api/tests/fixtures/Transportation Form.xml @@ -1,5 +1,5 @@ - + transportation_2011_07_25 @@ -39,6 +39,9 @@ + + + @@ -46,18 +49,83 @@ + + + + ambulance + + + + bicycle + + + + boat_canoe + + + + bus + + + + donkey_mule_cart + + + + keke_pepe + + + + lorry + + + + motorbike + + + + taxi + + + + other + + + + + + + + daily + + + + weekly + + + + other + + + + dont_know + + + + - - - - - - - - - - - + + + + + + + + + + + + @@ -66,46 +134,10 @@ @@ -115,198 +147,100 @@ - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + + + + + + + + + + diff --git a/kobo/apps/openrosa/apps/main/tests/fixtures/transportation/transportation.xml b/kobo/apps/openrosa/apps/main/tests/fixtures/transportation/transportation.xml index 4305eaaeca..795d62af52 100644 --- a/kobo/apps/openrosa/apps/main/tests/fixtures/transportation/transportation.xml +++ b/kobo/apps/openrosa/apps/main/tests/fixtures/transportation/transportation.xml @@ -1,5 +1,4 @@ - - + transportation_2011_07_25 @@ -39,6 +38,9 @@ + + + @@ -46,19 +48,84 @@ + + + + ambulance + + + + bicycle + + + + boat_canoe + + + + bus + + + + donkey_mule_cart + + + + keke_pepe + + + + lorry + + + + motorbike + + + + taxi + + + + other + + + + + + + + daily + + + + weekly + + + + other + + + + dont_know + + + + - - - - - - - - - - - - + + + + + + + + + + + + + @@ -66,46 +133,10 @@ @@ -115,198 +146,100 @@ - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + - - - daily - - - - weekly - - - - other - - - - dont_know - + + + + + + + + + + + + diff --git a/kpi/tests/test_asset_snapshots.py b/kpi/tests/test_asset_snapshots.py index 724db532c4..54b8020b01 100644 --- a/kpi/tests/test_asset_snapshots.py +++ b/kpi/tests/test_asset_snapshots.py @@ -78,7 +78,7 @@ def test_snapshots_allow_choice_duplicates(self): 'settings': {}, } snap = AssetSnapshot.objects.create(source=content) - assert snap.xml.count('ABC') == 2 + assert snap.xml.count('ABC') == 2 class AssetSnapshotHousekeeping(AssetSnapshotsTestCase): From f129a0a32b57434bce52e1291d72563050437033 Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Tue, 22 Oct 2024 13:32:15 -0400 Subject: [PATCH 3/6] update pyxform version to 2.1.1 --- dependencies/pip/dev_requirements.txt | 4 ++-- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 4 ++-- kobo/apps/openrosa/apps/main/tests/test_past_bugs.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index c48651a35d..cc6ad303e2 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -409,7 +409,7 @@ oauthlib==3.2.2 # -r dependencies/pip/requirements.in # django-oauth-toolkit # requests-oauthlib -openpyxl==3.1.2 +openpyxl==3.1.3 # via # -r dependencies/pip/requirements.in # pyxform @@ -530,7 +530,7 @@ pytz==2024.1 # via # flower # pandas -pyxform==2.0.3 +pyxform==2.1.1 # via # -r dependencies/pip/requirements.in # formpack diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 542f37120b..9979dfd6d2 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -77,7 +77,7 @@ openpyxl psycopg pymongo python-dateutil -pyxform==2.0.3 +pyxform==2.1.1 requests regex responses diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 9b38538fa8..506a180f0d 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -335,7 +335,7 @@ oauthlib==3.2.2 # -r dependencies/pip/requirements.in # django-oauth-toolkit # requests-oauthlib -openpyxl==3.1.2 +openpyxl==3.1.3 # via # -r dependencies/pip/requirements.in # pyxform @@ -411,7 +411,7 @@ pytz==2024.1 # via # flower # pandas -pyxform==2.0.3 +pyxform==2.1.1 # via # -r dependencies/pip/requirements.in # formpack diff --git a/kobo/apps/openrosa/apps/main/tests/test_past_bugs.py b/kobo/apps/openrosa/apps/main/tests/test_past_bugs.py index 156256a74d..5a1ad0c6ed 100644 --- a/kobo/apps/openrosa/apps/main/tests/test_past_bugs.py +++ b/kobo/apps/openrosa/apps/main/tests/test_past_bugs.py @@ -16,7 +16,7 @@ def test_uniqueness_of_group_names_enforced(self): self._create_user_and_login() response = self._publish_xls_file( 'fixtures/group_names_must_be_unique.xls') - message = 'The name "group_names_must_be_unique" is the same as the form name' + message = "The name 'group_names_must_be_unique' is the same as the form name" self.assertTrue(message in response.json()['text']) self.assertEqual(XForm.objects.count(), pre_count) From 97e6032f79dee37bab29e999be203c51e5861891 Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Wed, 23 Oct 2024 11:21:45 -0400 Subject: [PATCH 4/6] Ensure file is read correctly in by adding a check for in the save method --- .../apps/viewer/models/data_dictionary.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py b/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py index 2fba550631..79a69e4b88 100644 --- a/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py +++ b/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py @@ -158,19 +158,22 @@ def add_instances(self): def save(self, *args, **kwargs): if self.xls: file_obj = self.xls.read() - # Wrap the file data in a BytesIO object to simulate a file - xls_bytes_io = io.BytesIO(file_obj) - xls_bytes_io.name = self.xls.name - - survey = create_survey_from_xls(xls_bytes_io) - survey.update({ - 'name': survey.id_string, - }) - self.json = survey.to_json() - self.xml = survey.to_xml() - self._mark_start_time_boolean() - set_uuid(self) - self.set_uuid_in_xml(id_string=survey.id_string) + if file_obj: + # Wrap the file data in a BytesIO object to simulate a file + xls_bytes_io = io.BytesIO(file_obj) + xls_bytes_io.name = self.xls.name + + survey = create_survey_from_xls(xls_bytes_io) + survey.update( + { + 'name': survey.id_string, + } + ) + self.json = survey.to_json() + self.xml = survey.to_xml() + self._mark_start_time_boolean() + set_uuid(self) + self.set_uuid_in_xml(id_string=survey.id_string) super().save(*args, **kwargs) def file_name(self): From 5f66b6122470edea380385fa3c5b42e7b40d60ad Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Thu, 24 Oct 2024 11:33:16 -0400 Subject: [PATCH 5/6] pin pyxform version to commit that includes support for background-geopoint questions --- dependencies/pip/dev_requirements.txt | 8 ++++---- dependencies/pip/requirements.in | 4 +++- dependencies/pip/requirements.txt | 8 ++++---- jsapp/xlform/src/model.row.coffee | 1 + 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index 076b32a089..91cb4c6da8 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -12,6 +12,10 @@ # via -r dependencies/pip/requirements.in -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/XLSForm/pyxform.git@a724215f67c53bf529678125db15cae99cf61493#egg=pyxform + # via + # -r dependencies/pip/requirements.in + # formpack -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in aiohttp==3.9.3 @@ -537,10 +541,6 @@ pytz==2024.1 # via # flower # pandas -pyxform==2.1.1 - # via - # -r dependencies/pip/requirements.in - # formpack pyyaml==6.0.1 # via responses redis==5.0.3 diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 8f179255ad..6074fbd6dd 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -16,6 +16,9 @@ # using a fork for django > 1.1.0 compatibility -e git+https://github.com/trevoriancox/django-dont-vary-on.git@01a804122b7ddcdc22f50b40993f91c27b03bef6#egg=django-dont-vary-on +# pyxform +-e git+https://github.com/XLSForm/pyxform.git@a724215f67c53bf529678125db15cae99cf61493#egg=pyxform + # Regular PyPI packages Django>=4.2,<4.3 Markdown @@ -75,7 +78,6 @@ openpyxl psycopg pymongo python-dateutil -pyxform==2.1.1 requests regex responses diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index bcf2400265..83318025d0 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -12,6 +12,10 @@ # via -r dependencies/pip/requirements.in -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in +-e git+https://github.com/XLSForm/pyxform.git@a724215f67c53bf529678125db15cae99cf61493#egg=pyxform + # via + # -r dependencies/pip/requirements.in + # formpack -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in aiohttp==3.9.3 @@ -412,10 +416,6 @@ pytz==2024.1 # via # flower # pandas -pyxform==2.1.1 - # via - # -r dependencies/pip/requirements.in - # formpack pyyaml==6.0.1 # via responses redis==5.0.3 diff --git a/jsapp/xlform/src/model.row.coffee b/jsapp/xlform/src/model.row.coffee index 1829ea1ae1..8fefe705da 100644 --- a/jsapp/xlform/src/model.row.coffee +++ b/jsapp/xlform/src/model.row.coffee @@ -14,6 +14,7 @@ $skipLogicHelpers = require './mv.skipLogicHelpers' readParameters = require('../../js/components/formBuilder/formBuilderUtils').readParameters writeParameters = require('../../js/components/formBuilder/formBuilderUtils').writeParameters txtid = require('js/utils').txtid +notify = require('js/utils').notify module.exports = do -> row = {} From 72e535f5b01af93ecbf3246be31ab282af99b3aa Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Thu, 7 Nov 2024 00:00:30 -0500 Subject: [PATCH 6/6] Fix merge --- .../apps/viewer/models/data_dictionary.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py b/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py index 0c76e771c9..6a9922315a 100644 --- a/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py +++ b/kobo/apps/openrosa/apps/viewer/models/data_dictionary.py @@ -1,5 +1,4 @@ # coding: utf-8 -import io import os import re from xml.dom import Node @@ -165,15 +164,11 @@ def save(self, *args, **kwargs): ) if survey.name == DEFAULT_SURVEY_NAME: survey.name = survey.id_string - survey = create_survey_from_xls(xls_bytes_io) - survey.update( - { - ) - self.json = survey.to_json() - self.xml = survey.to_xml() - self._mark_start_time_boolean() - set_uuid(self) - self.set_uuid_in_xml(id_string=survey.id_string) + self.json = survey.to_json() + self.xml = survey.to_xml() + self._mark_start_time_boolean() + set_uuid(self) + self.set_uuid_in_xml(id_string=survey.id_string) super().save(*args, **kwargs) def file_name(self):