From 6e7c46d7137e72c7e44c3180c215bba58eef1c70 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Mon, 15 Jan 2024 22:31:13 +1100 Subject: [PATCH 01/22] dev: update dev dependencies and build process to current fashion - config: old [setup.cfg, setup.py], new [pyproject.toml] - format/lint: old [black, flake8, isort], new [ruff] - build/publish: old [build, twine], new [flit] - updated github actions and readme accordingly - source changes according to new default formatter and linter rules --- .github/workflows/release.yml | 23 ++--- .github/workflows/verify.yml | 10 +-- README.rst | 28 +++--- dev_requirements.in | 13 --- dev_requirements.pip | 42 --------- pre-commit.sh | 13 +-- pyproject.toml | 90 +++++++++++++++++-- pyxform/builder.py | 16 ++-- pyxform/survey.py | 4 - pyxform/survey_element.py | 26 +++--- pyxform/utils.py | 8 +- pyxform/validators/updater.py | 17 ++-- pyxform/xform2json.py | 2 +- pyxform/xform_instance_parser.py | 12 +-- pyxform/xls2json.py | 25 +++--- setup.cfg | 9 -- setup.py | 33 ------- tests/builder_tests.py | 1 + tests/dump_and_load_tests.py | 1 + tests/file_utils_test.py | 1 + tests/group_test.py | 1 + tests/j2x_question_tests.py | 1 + tests/j2x_test_creation.py | 1 + tests/j2x_test_instantiation.py | 1 + tests/j2x_test_xform_build_preparation.py | 1 - tests/loop_tests.py | 1 + tests/pyxform_test_case.py | 2 +- tests/settings_test.py | 34 +++---- tests/test_dynamic_default.py | 10 +-- tests/test_external_instances.py | 9 +- tests/test_external_instances_for_selects.py | 1 + tests/test_settings_auto_send_delete.py | 5 -- tests/test_survey.py | 4 +- tests/test_translations.py | 4 +- tests/test_validate_unicode_exception.py | 2 +- tests/test_validator_update.py | 2 +- tests/test_validator_util.py | 1 + tests/test_xform2json.py | 1 + tests/test_xls2json.py | 2 +- tests/test_xls2json_backends.py | 2 +- tests/tutorial_test.py | 1 + tests/utils.py | 1 + tests/xform2json_test.py | 2 +- tests/xform_test_case/attributecolumnstest.py | 2 +- tests/xform_test_case/bug_tests.py | 1 + tests/xform_test_case/xlsform_spec_test.py | 1 + tests/xform_test_case/xml_tests.py | 1 + tests/xls2json_tests.py | 1 + tests/xls2xform_tests.py | 1 + tests/xpath_helpers/choices.py | 4 +- tests/xpath_helpers/questions.py | 14 +-- 51 files changed, 226 insertions(+), 262 deletions(-) delete mode 100644 dev_requirements.in delete mode 100644 dev_requirements.pip delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c6a2f43b..26f7d5437 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,26 +24,19 @@ jobs: id: python-cache with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev_requirements.pip') }} + key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }} - name: Install dependencies. run: | python -m pip install --upgrade pip - pip install -r dev_requirements.pip + pip install -e . pip list - # Build. - - name: Build sdist and wheel. - run: | - pip install wheel - python clean_for_build.py - python setup.py sdist bdist_wheel - - # Publish. - - name: Publish release to PyPI with twine + # Build and publish. + - name: Publish release to PyPI if: success() run: | - pip install twine - twine upload dist/pyxform-*-py3-none-any.whl dist/pyxform-*.tar.gz + pip install flit==3.9.0 + flit --debug publish --no-use-vcs env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + FLIT_USERNAME: __token__ + FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 41ee83f56..4033be7ba 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -22,18 +22,16 @@ jobs: id: python-cache with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev_requirements.pip') }} + key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }} - name: Install dependencies. run: | python -m pip install --upgrade pip pip install -r dev_requirements.pip pip list - # Linters. - - run: black pyxform tests --check --diff - - run: isort pyxform tests --check-only --diff - - run: flake8 pyxform tests - - run: pycodestyle pyxform tests + # Linter. + - run: ruff format pyxform tests --diff + - run: ruff check pyxform tests --no-fix test: runs-on: ${{ matrix.os }} diff --git a/README.rst b/README.rst index c3dc2793a..ef9b4641b 100644 --- a/README.rst +++ b/README.rst @@ -50,6 +50,7 @@ From the command line, complete the following. These steps use a `virtualenv =3.2,<4"] +build-backend = "flit_core.buildapi" + +[tool.flit.module] +name = "pyxform" + +[tool.flit.sdist] +exclude = ["docs", "tests"] + +[tool.ruff] line-length = 90 +target-version = "py38" +fix = true +show-fixes = true +show-source = true +src = ["pyxform", "tests"] -[tool.isort] -profile = "black" -multi_line_output = 3 -line_length = 90 -default_section = "THIRDPARTY" -known_first_party = "pyxform" -sections = ['FUTURE', 'STDLIB', 'THIRDPARTY', 'FIRSTPARTY', 'LOCALFOLDER'] +[tool.ruff.lint] +# By default, ruff enables flake8's F rules, along with a subset of the E rules. +extend-select = [ + "I", # isort +] +# Potentially useful rule sets to enable in future: +#select = [ +# "A", # flake8-builtins +# "B", # flake8-bugbear +# "C4", # flake8-comprehensions +# "E", # pycodestyle error +# "ERA", # eradicate (commented out code) +# "F", # pyflakes +# "I", # isort +# "PERF", # perflint +# "PIE", # flake8-pie +# "PL", # pylint +# "PTH", # flake8-use-pathlib +# "PYI", # flake8-pyi +# "RET", # flake8-return +# "RUF", # ruff-specific rules +# "S", # flake8-bandit +# "SIM", # flake8-simplify +# "TRY", # tryceratops +# "UP", # pyupgrade +# "W", # pycodestyle warning +#] +ignore = [ + "E501", # line-too-long (we have a lot of long strings) + "F821", # undefined-name (doesn't work well with type hints as of ruff 0.1.11). + "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) +] diff --git a/pyxform/builder.py b/pyxform/builder.py index 8837593ad..285c0c8e3 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -42,15 +42,15 @@ def copy_json_dict(json_dict): json_dict_copy = None items = None - if type(json_dict) is list: + if isinstance(json_dict, list): json_dict_copy = [None] * len(json_dict) items = enumerate(json_dict) - elif type(json_dict) is dict: + elif isinstance(json_dict, dict): json_dict_copy = {} items = json_dict.items() for key, value in items: - if type(value) is dict or type(value) is list: + if isinstance(value, (dict, list)): json_dict_copy[key] = copy_json_dict(value) else: json_dict_copy[key] = value @@ -97,7 +97,7 @@ def set_sections(self, sections): the name of the section and the value is a dict that can be used to create a whole survey. """ - assert type(sections) == dict + assert isinstance(sections, dict) self._sections = sections def create_survey_element_from_dict( @@ -336,7 +336,7 @@ def _name_and_label_substitutions(question_template, column_headers): # if the label in column_headers has multiple languages setup a # dictionary by language to do substitutions. info_by_lang = {} - if type(column_headers[const.LABEL]) == dict: + if isinstance(column_headers[const.LABEL], dict): info_by_lang = dict( [ ( @@ -352,12 +352,12 @@ def _name_and_label_substitutions(question_template, column_headers): result = question_template.copy() for key in result.keys(): - if type(result[key]) == str: + if isinstance(result[key], str): result[key] %= column_headers - elif type(result[key]) == dict: + elif isinstance(result[key], dict): result[key] = result[key].copy() for key2 in result[key].keys(): - if type(column_headers[const.LABEL]) == dict: + if isinstance(column_headers[const.LABEL], dict): result[key][key2] %= info_by_lang.get(key2, column_headers) else: result[key][key2] %= column_headers diff --git a/pyxform/survey.py b/pyxform/survey.py index a45659b1f..a56aeb466 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -835,7 +835,6 @@ def _set_up_media_translations(media_dict, translation_key): media_dict = media_dict_default for media_type, possibly_localized_media in media_dict.items(): - if media_type not in SurveyElement.SUPPORTED_MEDIA: raise PyXFormError("Media type: " + media_type + " not supported") @@ -848,7 +847,6 @@ def _set_up_media_translations(media_dict, translation_key): localized_media = {self.default_language: possibly_localized_media} for language, media in localized_media.items(): - # Create the required dictionaries in _translations, # then add media as a leaf value: if language not in self._translations: @@ -1039,14 +1037,12 @@ def _is_return_relative_path() -> bool: current_matchobj = matchobj if not last_saved and context: - if not is_indexed_repeat: return True # It is possible to have multiple indexed-repeat in an expression indexed_repeats_iter = RE_INDEXED_REPEAT.finditer(matchobj.string) for indexed_repeat in indexed_repeats_iter: - # Make sure current ${name} is in the correct indexed-repeat if current_matchobj.end() > indexed_repeat.end(): try: diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index 9a11508ea..e20238efa 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -27,7 +27,7 @@ def _overlay(over, under): - if type(under) == dict: + if isinstance(under, dict): result = under.copy() result.update(over) return result @@ -123,7 +123,7 @@ def add_child(self, child): child.parent = self def add_children(self, children): - if type(children) == list: + if isinstance(children, list): for child in children: self.add_child(child) else: @@ -282,9 +282,9 @@ def get_translations(self, default_language): the block. @see survey._setup_translations """ bind_dict = self.get("bind") - if bind_dict and type(bind_dict) is dict: + if bind_dict and isinstance(bind_dict, dict): constraint_msg = bind_dict.get("jr:constraintMsg") - if type(constraint_msg) is dict: + if isinstance(constraint_msg, dict): for lang, text in constraint_msg.items(): yield { "path": self._translation_path("jr:constraintMsg"), @@ -301,7 +301,7 @@ def get_translations(self, default_language): } required_msg = bind_dict.get("jr:requiredMsg") - if type(required_msg) is dict: + if isinstance(required_msg, dict): for lang, text in required_msg.items(): yield { "path": self._translation_path("jr:requiredMsg"), @@ -317,7 +317,7 @@ def get_translations(self, default_language): "output_context": self, } no_app_error_string = bind_dict.get("jr:noAppErrorString") - if type(no_app_error_string) is dict: + if isinstance(no_app_error_string, dict): for lang, text in no_app_error_string.items(): yield { "path": self._translation_path("jr:noAppErrorString"), @@ -332,7 +332,7 @@ def get_translations(self, default_language): if ( display_element == "label" and self.needs_itext_ref() - and type(label_or_hint) is not dict + and not isinstance(label_or_hint, dict) and label_or_hint ): label_or_hint = {default_language: label_or_hint} @@ -356,7 +356,7 @@ def get_translations(self, default_language): ): label_or_hint = {default_language: label_or_hint} - if type(label_or_hint) is dict: + if isinstance(label_or_hint, dict): for lang, text in label_or_hint.items(): yield { "display_element": display_element, # Not used @@ -375,8 +375,8 @@ def get_media_keys(self): return {"media": "%s:media" % self.get_xpath()} def needs_itext_ref(self): - return type(self.label) is dict or ( - type(self.media) is dict and len(self.media) > 0 + return isinstance(self.label, dict) or ( + isinstance(self.media, dict) and len(self.media) > 0 ) def get_setvalue_node_for_dynamic_default(self, in_repeat=False): @@ -472,14 +472,14 @@ def xml_bindings(self): ): v = self.BINDING_CONVERSIONS[v] if k == "jr:constraintMsg" and ( - type(v) is dict or re.search(BRACKETED_TAG_REGEX, v) + isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) ): v = "jr:itext('%s')" % self._translation_path("jr:constraintMsg") if k == "jr:requiredMsg" and ( - type(v) is dict or re.search(BRACKETED_TAG_REGEX, v) + isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) ): v = "jr:itext('%s')" % self._translation_path("jr:requiredMsg") - if k == "jr:noAppErrorString" and type(v) is dict: + if k == "jr:noAppErrorString" and isinstance(v, dict): v = "jr:itext('%s')" % self._translation_path("jr:noAppErrorString") bind_dict[k] = survey.insert_xpaths(v, context=self) return [node("bind", nodeset=self.get_xpath(), **bind_dict)] diff --git a/pyxform/utils.py b/pyxform/utils.py index 70bb194d8..968f2d062 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -111,7 +111,7 @@ def node(*args, **kwargs) -> DetachableElement: tag = args[0] if len(args) > 0 else kwargs["tag"] args = args[1:] result = DetachableElement(tag) - unicode_args = [u for u in args if type(u) == str] + unicode_args = [u for u in args if isinstance(u, str)] assert len(unicode_args) <= 1 parsed_string = False @@ -147,11 +147,11 @@ def node(*args, **kwargs) -> DetachableElement: text_node.data = unicode_args[0] result.appendChild(text_node) for n in args: - if type(n) == int or type(n) == float or type(n) == bytes: + if isinstance(n, (int, float, bytes)): text_node = PatchedText() text_node.data = str(n) result.appendChild(text_node) - elif type(n) is not str: + elif not isinstance(n, str): result.appendChild(n) return result @@ -271,7 +271,7 @@ def get_languages_with_bad_tags(languages): lang_code = re.search(lang_code_regex, lang) if lang != "default" and ( - not lang_code or not lang_code.group(1) in iana_subtags + not lang_code or lang_code.group(1) not in iana_subtags ): languages_with_bad_tags.append(lang) return languages_with_bad_tags diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index 73e5994ba..4a834651b 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -221,8 +221,9 @@ def _find_download_url(update_info, json_data, file_name): if len(files) == 0: raise PyXFormError( - "No files attached to release '{r}'.\n\n{h}" - "".format(r=rel_name, h=update_info.manual_msg) + "No files attached to release '{r}'.\n\n{h}" "".format( + r=rel_name, h=update_info.manual_msg + ) ) file_urls = [x["browser_download_url"] for x in files if x["name"] == file_name] @@ -270,8 +271,9 @@ def _get_bin_paths(update_info, file_path): main_bin = "*validate" else: raise PyXFormError( - "Did not find a supported main binary for file: {p}.\n\n{h}" - "".format(p=file_path, h=update_info.manual_msg) + "Did not find a supported main binary for file: {p}.\n\n{h}" "".format( + p=file_path, h=update_info.manual_msg + ) ) return [ (main_bin, update_info.validator_basename), @@ -309,8 +311,9 @@ def _unzip_find_jobs(open_zip_file, bin_paths, out_path): zip_jobs[file_out_path] = zip_item if len(bin_paths) != len(zip_jobs.keys()): raise PyXFormError( - "Expected {e} zip job files, found: {c}" - "".format(e=len(bin_paths), c=len(zip_jobs.keys())) + "Expected {e} zip job files, found: {c}" "".format( + e=len(bin_paths), c=len(zip_jobs.keys()) + ) ) return zip_jobs @@ -497,7 +500,6 @@ def check(update_info): class _UpdateService: - update_info = None def list(self): @@ -556,7 +558,6 @@ def _install_check(bin_file_path=None): def _build_validator_menu(main_subparser, validator_name, updater_instance): - main = main_subparser.add_parser( validator_name.lower(), description="{v} Sub-menu".format(v=validator_name), diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 4cf0ebccf..f0b5e8aac 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -78,7 +78,7 @@ def _convert_dict_to_xml_recurse(parent, dictitem): assert not isinstance(dictitem, list) if isinstance(dictitem, dict): - for (tag, child) in iter(dictitem.items()): + for tag, child in iter(dictitem.items()): if str(tag) == "_text": parent.text = str(child) elif isinstance(child, list): diff --git a/pyxform/xform_instance_parser.py b/pyxform/xform_instance_parser.py index d6e31e2c6..93d4a9afc 100644 --- a/pyxform/xform_instance_parser.py +++ b/pyxform/xform_instance_parser.py @@ -29,7 +29,7 @@ def _xml_node_to_dict(node): if child_name not in value: # copy the value into the dict value[child_name] = d[child_name] - elif type(value[child_name]) == list: + elif isinstance(value[child_name], list): # add to the existing list value[child_name].append(d[child_name]) else: @@ -42,15 +42,15 @@ def _flatten_dict(d, prefix): """ Return a list of XPath, value pairs. """ - assert type(d) == dict - assert type(prefix) == list + assert isinstance(d, dict) + assert isinstance(prefix, list) for key, value in d.items(): new_prefix = prefix + [key] - if type(value) == dict: + if isinstance(value, dict): for pair in _flatten_dict(value, new_prefix): yield pair - elif type(value) == list: + elif isinstance(value, list): for i, item in enumerate(value): item_prefix = list(new_prefix) # make a copy # note on indexing xpaths: IE5 and later has @@ -58,7 +58,7 @@ def _flatten_dict(d, prefix): # according to the W3C standard it should have been # [1]. I'm adding 1 to i to start at 1. item_prefix[-1] += "[%s]" % str(i + 1) - if type(item) == dict: + if isinstance(item, dict): for pair in _flatten_dict(item, item_prefix): yield pair else: diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 02fe8aa2f..c15f915d1 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -252,7 +252,7 @@ def has_double_colon(workbook_dict) -> bool: for sheet in workbook_dict.values(): for row in sheet: for column_header in row.keys(): - if type(column_header) is not str: + if not isinstance(column_header, str): continue if "::" in column_header: return True @@ -782,8 +782,9 @@ def workbook_to_json( new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update( { - "odk:" - + constants.TRACK_CHANGES: parameters[constants.TRACK_CHANGES] + "odk:" + constants.TRACK_CHANGES: parameters[ + constants.TRACK_CHANGES + ] } ) @@ -811,8 +812,9 @@ def workbook_to_json( new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update( { - "odk:" - + constants.IDENTIFY_USER: parameters[constants.IDENTIFY_USER] + "odk:" + constants.IDENTIFY_USER: parameters[ + constants.IDENTIFY_USER + ] } ) @@ -823,7 +825,6 @@ def workbook_to_json( ) if any(k in parameters.keys() for k in location_parameters): if all(k in parameters.keys() for k in location_parameters): - if parameters[constants.LOCATION_PRIORITY] not in [ "no-power", "low-power", @@ -882,16 +883,13 @@ def workbook_to_json( new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update( { - "odk:" - + constants.LOCATION_MAX_AGE: parameters[ + "odk:" + constants.LOCATION_MAX_AGE: parameters[ constants.LOCATION_MAX_AGE ], - "odk:" - + constants.LOCATION_MIN_INTERVAL: parameters[ + "odk:" + constants.LOCATION_MIN_INTERVAL: parameters[ constants.LOCATION_MIN_INTERVAL ], - "odk:" - + constants.LOCATION_PRIORITY: parameters[ + "odk:" + constants.LOCATION_PRIORITY: parameters[ constants.LOCATION_PRIORITY ], } @@ -1127,7 +1125,8 @@ def workbook_to_json( and constants.CHOICE_FILTER not in row ): warnings.append( - ROW_FORMAT_STRING % row_number + ROW_FORMAT_STRING + % row_number + " select one external is only meant for" " filtered selects." ) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d3eef26d6..000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -# Make sure line length matches pyproject.toml setting for black -max-line-length=90 -ignore=E203,E501,W503 - -[pycodestyle] -# Make sure line length matches pyproject.toml setting for black -max-line-length=90 -ignore=E203,E501,W503 diff --git a/setup.py b/setup.py deleted file mode 100644 index fec5a1fdf..000000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -""" -pyxform - Python library that converts XLSForms to XForms. -""" -from setuptools import find_packages, setup - - -setup( - name="pyxform", - version="2.0.2", - author="github.com/xlsform", - author_email="info@xlsform.org", - packages=find_packages(exclude=["tests", "tests.*"]), - package_data={ - "pyxform.validators.odk_validate": ["bin/*.*"], - "pyxform": ["iana_subtags.txt"], - }, - url="http://pypi.python.org/pypi/pyxform/", - description="A Python package to create XForms for ODK Collect.", - long_description=open("README.rst", "rt").read(), - python_requires=">=3.7", - install_requires=[ - "xlrd==2.0.1", - "openpyxl==3.0.9", - "defusedxml==0.7.1", - ], - entry_points={ - "console_scripts": [ - "xls2xform=pyxform.xls2xform:main_cli", - "pyxform_validator_update=pyxform.validators.updater:main_cli", - ] - }, -) diff --git a/tests/builder_tests.py b/tests/builder_tests.py index 19cfaed3f..a87bd63bb 100644 --- a/tests/builder_tests.py +++ b/tests/builder_tests.py @@ -11,6 +11,7 @@ from pyxform.builder import SurveyElementBuilder, create_survey_from_xls from pyxform.errors import PyXFormError from pyxform.xls2json import print_pyobj_to_json + from tests import utils FIXTURE_FILETYPE = "xls" diff --git a/tests/dump_and_load_tests.py b/tests/dump_and_load_tests.py index 8bdd2089b..2ec047503 100644 --- a/tests/dump_and_load_tests.py +++ b/tests/dump_and_load_tests.py @@ -6,6 +6,7 @@ from unittest import TestCase from pyxform.builder import create_survey_from_path + from tests import utils diff --git a/tests/file_utils_test.py b/tests/file_utils_test.py index a697ed4c7..cb8bd9b3c 100644 --- a/tests/file_utils_test.py +++ b/tests/file_utils_test.py @@ -5,6 +5,7 @@ from unittest import TestCase from pyxform.xls2json_backends import convert_file_to_csv_string + from tests import utils diff --git a/tests/group_test.py b/tests/group_test.py index 37fb69527..2455ece67 100644 --- a/tests/group_test.py +++ b/tests/group_test.py @@ -6,6 +6,7 @@ from pyxform.builder import create_survey_element_from_dict from pyxform.xls2json import SurveyReader + from tests import utils diff --git a/tests/j2x_question_tests.py b/tests/j2x_question_tests.py index f04657ba2..35c3e24e7 100644 --- a/tests/j2x_question_tests.py +++ b/tests/j2x_question_tests.py @@ -6,6 +6,7 @@ from pyxform import Survey from pyxform.builder import create_survey_element_from_dict + from tests.utils import prep_class_config TESTING_BINDINGS = True diff --git a/tests/j2x_test_creation.py b/tests/j2x_test_creation.py index 0843ee009..af0a303fd 100644 --- a/tests/j2x_test_creation.py +++ b/tests/j2x_test_creation.py @@ -5,6 +5,7 @@ from unittest import TestCase from pyxform import MultipleChoiceQuestion, Survey, create_survey_from_xls + from tests import utils diff --git a/tests/j2x_test_instantiation.py b/tests/j2x_test_instantiation.py index 5749badc8..d4c1f5736 100644 --- a/tests/j2x_test_instantiation.py +++ b/tests/j2x_test_instantiation.py @@ -6,6 +6,7 @@ from pyxform import Survey, SurveyInstance from pyxform.builder import create_survey_element_from_dict + from tests.utils import prep_class_config diff --git a/tests/j2x_test_xform_build_preparation.py b/tests/j2x_test_xform_build_preparation.py index cec5f4f79..f3b4ca5b6 100644 --- a/tests/j2x_test_xform_build_preparation.py +++ b/tests/j2x_test_xform_build_preparation.py @@ -9,7 +9,6 @@ class Json2XformExportingPrepTests(TestCase): def test_dictionary_consolidates_duplicate_entries(self): - yes_or_no_dict_array = [ {"label": {"French": "Oui", "English": "Yes"}, "name": "yes"}, {"label": {"French": "Non", "English": "No"}, "name": "no"}, diff --git a/tests/loop_tests.py b/tests/loop_tests.py index 55b1dfbf3..e6a52a609 100644 --- a/tests/loop_tests.py +++ b/tests/loop_tests.py @@ -5,6 +5,7 @@ from unittest import TestCase from pyxform.builder import create_survey_from_xls + from tests import utils diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 3849ac523..22d2f2dbf 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -15,12 +15,12 @@ # noinspection PyProtectedMember from lxml.etree import _Element - from pyxform.builder import create_survey_element_from_dict from pyxform.errors import PyXFormError from pyxform.utils import NSMAP, coalesce from pyxform.validators.odk_validate import ODKValidateError, check_xform from pyxform.xls2json import workbook_to_json + from tests.test_utils.md_table import md_table_to_ss_structure logger = logging.getLogger(__name__) diff --git a/tests/settings_test.py b/tests/settings_test.py index 85ebdf979..4bf2289a3 100644 --- a/tests/settings_test.py +++ b/tests/settings_test.py @@ -7,11 +7,11 @@ from pyxform.builder import create_survey_from_path from pyxform.xls2json import SurveyReader + from tests import utils class SettingsTests(TestCase): - maxDiff = None def setUp(self): @@ -20,26 +20,26 @@ def setUp(self): def test_survey_reader(self): survey_reader = SurveyReader(self.path, default_name="settings") expected_dict = { - u"id_string": u"new_id", - u"sms_keyword": u"new_id", - u"default_language": u"default", - u"name": u"settings", - u"title": u"My Survey", - u"type": u"survey", - u"attribute": { - u"my_number": u"1234567890", - u"my_string": u"lor\xe9m ipsum", + "id_string": "new_id", + "sms_keyword": "new_id", + "default_language": "default", + "name": "settings", + "title": "My Survey", + "type": "survey", + "attribute": { + "my_number": "1234567890", + "my_string": "lor\xe9m ipsum", }, - u"children": [ + "children": [ { - u"name": u"your_name", - u"label": {u"english": u"What is your name?"}, - u"type": u"text", + "name": "your_name", + "label": {"english": "What is your name?"}, + "type": "text", }, { - u"name": u"your_age", - u"label": {u"english": u"How many years old are you?"}, - u"type": u"integer", + "name": "your_age", + "label": {"english": "How many years old are you?"}, + "type": "integer", }, { "children": [ diff --git a/tests/test_dynamic_default.py b/tests/test_dynamic_default.py index cd6642291..268f55360 100644 --- a/tests/test_dynamic_default.py +++ b/tests/test_dynamic_default.py @@ -10,8 +10,8 @@ from unittest.mock import patch import psutil - from pyxform import utils + from tests.pyxform_test_case import PyxformTestCase from tests.xpath_helpers.choices import xpc from tests.xpath_helpers.questions import xpq @@ -45,7 +45,7 @@ class XPathHelper: @staticmethod def model_setvalue(q_num: int): """Get the setvalue element's value attribute.""" - return fr""" + return rf""" /h:html/h:head/x:model/x:setvalue[ @ref="/test_name/q{q_num}" and @event='odk-instance-first-load' @@ -79,7 +79,7 @@ def model(q_num: int, case: Case): value_cmp = "" else: value_cmp = f"""and @value="{q_default_final}" """ - return fr""" + return rf""" /h:html/h:head/x:model /x:instance/x:test_name[@id="test_id"]/x:q{q_num}[ not(text()) @@ -102,7 +102,7 @@ def model(q_num: int, case: Case): q_default_cmp = "" else: q_default_cmp = f"""and text()='{q_default_final}' """ - return fr""" + return rf""" /h:html/h:head/x:model /x:instance/x:test_name[@id="test_id"]/x:q{q_num}[ ancestor::x:model/x:bind[ @@ -137,7 +137,7 @@ def body_select1(q_num: int, choices: Tuple[Tuple[str, str], ...]): for cv, cl in choices ) ) - return fr""" + return rf""" /h:html/h:body/x:select1[ @ref = '/test/q{q_num}' and ./x:label/text() = 'Select{q_num}' diff --git a/tests/test_external_instances.py b/tests/test_external_instances.py index 385cb8323..be23a67ce 100644 --- a/tests/test_external_instances.py +++ b/tests/test_external_instances.py @@ -7,6 +7,7 @@ from textwrap import dedent from pyxform.errors import PyXFormError + from tests.pyxform_test_case import PyxformTestCase, PyxformTestError from tests.xpath_helpers.choices import xpc @@ -339,7 +340,9 @@ def test_can__reuse_csv__pulldata_then_selects(self): | | select_one_from_file pain_locations.csv | pmonth | Location of worst pain this month. | | | | select_one_from_file pain_locations.csv | pyear | Location of worst pain this year. | | """ # noqa - expected = """""" # noqa + expected = ( + """""" # noqa + ) self.assertPyxformXform(md=md, model__contains=[expected]) def test_can__reuse_xml__selects_then_external(self): @@ -371,7 +374,9 @@ def test_can__reuse_xml__external_then_selects(self): | | select_one_from_file pain_locations.xml | pmonth | Location of worst pain this month. | | | select_one_from_file pain_locations.xml | pyear | Location of worst pain this year. | """ # noqa - expected = """""" # noqa + expected = ( + """""" # noqa + ) self.assertPyxformXform(md=md, model__contains=[expected]) survey = self.md_to_pyxform_survey(md_raw=md) xml = survey._to_pretty_xml() diff --git a/tests/test_external_instances_for_selects.py b/tests/test_external_instances_for_selects.py index 028ada445..1d39bc9a4 100644 --- a/tests/test_external_instances_for_selects.py +++ b/tests/test_external_instances_for_selects.py @@ -11,6 +11,7 @@ from pyxform.constants import EXTERNAL_INSTANCE_EXTENSIONS from pyxform.errors import PyXFormError from pyxform.xls2xform import get_xml_path, xls2xform_convert + from tests.pyxform_test_case import PyxformTestCase from tests.test_utils.md_table import md_table_to_workbook from tests.utils import get_temp_dir diff --git a/tests/test_settings_auto_send_delete.py b/tests/test_settings_auto_send_delete.py index ed88628f6..0b1e302d6 100644 --- a/tests/test_settings_auto_send_delete.py +++ b/tests/test_settings_auto_send_delete.py @@ -7,7 +7,6 @@ class SettingsAutoSendDelete(PyxformTestCase): def test_settings_auto_send_true(self): - self.assertPyxformXform( name="data", md=""" @@ -22,7 +21,6 @@ def test_settings_auto_send_true(self): ) def test_settings_auto_delete_true(self): - self.assertPyxformXform( name="data", md=""" @@ -37,7 +35,6 @@ def test_settings_auto_delete_true(self): ) def test_settings_auto_send_delete_false(self): - self.assertPyxformXform( name="data", md=""" @@ -52,7 +49,6 @@ def test_settings_auto_send_delete_false(self): ) def test_settings_without_submission_url_does_not_generate_method_attribute(self): - self.assertPyxformXform( name="data", md=""" @@ -75,7 +71,6 @@ def test_settings_without_submission_url_does_not_generate_method_attribute(self ) def test_settings_with_submission_url_generates_method_attribute(self): - self.assertPyxformXform( name="data", md=""" diff --git a/tests/test_survey.py b/tests/test_survey.py index 6718d1143..af4fb2374 100644 --- a/tests/test_survey.py +++ b/tests/test_survey.py @@ -15,9 +15,7 @@ def test_many_xpath_references_do_not_hit_64_recursion_limit__one_to_one(self): | | text | q1 | Q1 | | | | note | n | {n} | | | | text | q2 | Q2 | {r} | - """.format( - n="q1 = ${q1} " * 250, r=" or ".join(["${q1} = 'y'"] * 250) - ), + """.format(n="q1 = ${q1} " * 250, r=" or ".join(["${q1} = 'y'"] * 250)), ) def test_many_xpath_references_do_not_hit_64_recursion_limit__many_to_one(self): diff --git a/tests/test_translations.py b/tests/test_translations.py index 1346cf88f..8ab87a9ea 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -7,13 +7,13 @@ from time import perf_counter from unittest.mock import patch -from pyxform.constants import CHOICES +from pyxform.constants import CHOICES, SURVEY from pyxform.constants import DEFAULT_LANGUAGE_VALUE as DEFAULT_LANG -from pyxform.constants import SURVEY from pyxform.validators.pyxform.translations_checks import ( OR_OTHER_WARNING, format_missing_translations_msg, ) + from tests.pyxform_test_case import PyxformTestCase from tests.xpath_helpers.choices import xpc from tests.xpath_helpers.questions import xpq diff --git a/tests/test_validate_unicode_exception.py b/tests/test_validate_unicode_exception.py index d49fcde1a..fa93392dd 100644 --- a/tests/test_validate_unicode_exception.py +++ b/tests/test_validate_unicode_exception.py @@ -27,7 +27,7 @@ def test_validate_unicode_exception(self): def test_validate_with_more_unicode(self): self.assertPyxformXform( - md=u""" + md=""" | survey | | | | | | | type | name | label | calculation | | | calculate | bad | bad | £¥§©®₱₩ | diff --git a/tests/test_validator_update.py b/tests/test_validator_update.py index a311ec66b..735c1b533 100644 --- a/tests/test_validator_update.py +++ b/tests/test_validator_update.py @@ -17,6 +17,7 @@ _UpdateInfo, capture_handler, ) + from tests import validators from tests.utils import get_temp_dir, get_temp_file from tests.validators.server import ThreadingServerInThread @@ -77,7 +78,6 @@ def test_get_temp_dir(self): class TestUpdateHandler(TestCase): - server: "Optional[ThreadingServerInThread]" = None @classmethod diff --git a/tests/test_validator_util.py b/tests/test_validator_util.py index 42cbef3d1..e39a75141 100644 --- a/tests/test_validator_util.py +++ b/tests/test_validator_util.py @@ -7,6 +7,7 @@ from pyxform.validators.error_cleaner import ErrorCleaner from pyxform.validators.util import XFORM_SPEC_PATH, check_readable + from tests.utils import prep_class_config diff --git a/tests/test_xform2json.py b/tests/test_xform2json.py index 48efc0702..061190fe2 100644 --- a/tests/test_xform2json.py +++ b/tests/test_xform2json.py @@ -5,6 +5,7 @@ import json from pyxform.builder import create_survey_element_from_dict + from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_xls2json.py b/tests/test_xls2json.py index 4fce54528..b6a1370da 100644 --- a/tests/test_xls2json.py +++ b/tests/test_xls2json.py @@ -1,9 +1,9 @@ import os import psutil - from pyxform.xls2json_backends import xlsx_to_dict from pyxform.xls2xform import get_xml_path, xls2xform_convert + from tests import example_xls, test_output from tests.pyxform_test_case import PyxformTestCase from tests.test_utils.md_table import md_table_to_workbook diff --git a/tests/test_xls2json_backends.py b/tests/test_xls2json_backends.py index 50285c3b6..d3d17e488 100644 --- a/tests/test_xls2json_backends.py +++ b/tests/test_xls2json_backends.py @@ -8,13 +8,13 @@ import openpyxl import xlrd - from pyxform.xls2json_backends import ( xls_to_dict, xls_value_to_unicode, xlsx_to_dict, xlsx_value_to_str, ) + from tests import bug_example_xls, utils diff --git a/tests/tutorial_test.py b/tests/tutorial_test.py index eb84cb395..7a9dd5de5 100644 --- a/tests/tutorial_test.py +++ b/tests/tutorial_test.py @@ -5,6 +5,7 @@ from unittest import TestCase from pyxform.builder import create_survey_from_path + from tests import utils diff --git a/tests/utils.py b/tests/utils.py index e4dbd0771..d1cece88c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,6 +12,7 @@ from pyxform import file_utils from pyxform.builder import create_survey, create_survey_from_path + from tests import example_xls if TYPE_CHECKING: diff --git a/tests/xform2json_test.py b/tests/xform2json_test.py index 31f6a6702..90210fa04 100644 --- a/tests/xform2json_test.py +++ b/tests/xform2json_test.py @@ -8,13 +8,13 @@ from pyxform.builder import create_survey_from_path from pyxform.xform2json import _try_parse, create_survey_element_from_xml + from tests import test_output, utils from tests.pyxform_test_case import PyxformTestCase from tests.xform_test_case.base import XFormTestCase class DumpAndLoadXForm2JsonTests(XFormTestCase, PyxformTestCase): - maxDiff = None def setUp(self): diff --git a/tests/xform_test_case/attributecolumnstest.py b/tests/xform_test_case/attributecolumnstest.py index 1ad6cfe23..59fda3b50 100644 --- a/tests/xform_test_case/attributecolumnstest.py +++ b/tests/xform_test_case/attributecolumnstest.py @@ -7,12 +7,12 @@ import unittest import pyxform + from tests import test_expected_output from tests.xform_test_case.base import XFormTestCase class AttributeColumnsTest(XFormTestCase): - maxDiff = None def runTest(self): diff --git a/tests/xform_test_case/bug_tests.py b/tests/xform_test_case/bug_tests.py index 08bca787c..7f38f709d 100644 --- a/tests/xform_test_case/bug_tests.py +++ b/tests/xform_test_case/bug_tests.py @@ -11,6 +11,7 @@ from pyxform.validators.odk_validate import ODKValidateError, check_xform from pyxform.xls2json import SurveyReader, parse_file_to_workbook_dict from pyxform.xls2json_backends import xlsx_to_dict + from tests import bug_example_xls, example_xls, test_expected_output, test_output from tests.xform_test_case.base import XFormTestCase diff --git a/tests/xform_test_case/xlsform_spec_test.py b/tests/xform_test_case/xlsform_spec_test.py index 61d08b647..a1c9dd585 100644 --- a/tests/xform_test_case/xlsform_spec_test.py +++ b/tests/xform_test_case/xlsform_spec_test.py @@ -7,6 +7,7 @@ from pyxform import builder, xls2json from pyxform.errors import PyXFormError + from tests import example_xls, test_expected_output from tests.xform_test_case.base import XFormTestCase diff --git a/tests/xform_test_case/xml_tests.py b/tests/xform_test_case/xml_tests.py index 5908aa861..311d286e2 100644 --- a/tests/xform_test_case/xml_tests.py +++ b/tests/xform_test_case/xml_tests.py @@ -7,6 +7,7 @@ from pyxform import create_survey_from_xls from pyxform.utils import node + from tests.utils import path_to_text_fixture from tests.xform_test_case.base import XFormTestCase diff --git a/tests/xls2json_tests.py b/tests/xls2json_tests.py index 6e5360aa5..4a440a763 100644 --- a/tests/xls2json_tests.py +++ b/tests/xls2json_tests.py @@ -9,6 +9,7 @@ from pyxform.xls2json import SurveyReader from pyxform.xls2json_backends import csv_to_dict, xls_to_dict, xlsx_to_dict + from tests import example_xls, test_expected_output, test_output, utils diff --git a/tests/xls2xform_tests.py b/tests/xls2xform_tests.py index 5d9b8898b..c7d8119e6 100644 --- a/tests/xls2xform_tests.py +++ b/tests/xls2xform_tests.py @@ -17,6 +17,7 @@ get_xml_path, main_cli, ) + from tests.utils import path_to_text_fixture try: diff --git a/tests/xpath_helpers/choices.py b/tests/xpath_helpers/choices.py index d546221b9..4489199ca 100644 --- a/tests/xpath_helpers/choices.py +++ b/tests/xpath_helpers/choices.py @@ -22,7 +22,7 @@ def model_instance_choices_label(cname: str, choices: Tuple[Tuple[str, str], ... for cv, cl in choices ) ) - return fr""" + return rf""" /h:html/h:head/x:model/x:instance[@id='{cname}']/x:root[ {choices_xp} ] @@ -41,7 +41,7 @@ def model_instance_choices_itext(cname: str, choices: Tuple[str, ...]): for idx, cv in enumerate(choices) ) ) - return fr""" + return rf""" /h:html/h:head/x:model/x:instance[@id='{cname}']/x:root[ {choices_xp} ] diff --git a/tests/xpath_helpers/questions.py b/tests/xpath_helpers/questions.py index bf865ab35..95d31ba26 100644 --- a/tests/xpath_helpers/questions.py +++ b/tests/xpath_helpers/questions.py @@ -6,14 +6,14 @@ class XPathHelper: @staticmethod def model_instance_item(q_name: str): """Model instance contains the question item.""" - return fr""" + return rf""" /h:html/h:head/x:model/x:instance/x:test_name/x:{q_name} """ @staticmethod def model_instance_bind(q_name: str, _type: str): """Model instance contains the question item.""" - return fr""" + return rf""" /h:html/h:head/x:model/x:bind[ @nodeset='/test_name/{q_name}' and @type='{_type}' @@ -64,7 +64,7 @@ def body_label_itext(q_type: str, q_name: str): @staticmethod def body_select1_itemset(q_name: str): """Body has a select1 with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/x:select1[ @ref = '/test_name/{q_name}' and ./x:itemset @@ -75,7 +75,7 @@ def body_select1_itemset(q_name: str): @staticmethod def body_group_select1_itemset(g_name: str, q_name: str): """Body has a select1 with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/x:group[@ref='/test_name/{g_name}']/x:select1[ @ref = '/test_name/{g_name}/{q_name}' and ./x:itemset @@ -86,7 +86,7 @@ def body_group_select1_itemset(g_name: str, q_name: str): @staticmethod def body_repeat_select1_itemset(r_name: str, q_name: str): """Body has a select1 with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/x:group[@ref='/test_name/{r_name}'] /x:repeat[@nodeset='/test_name/{r_name}'] /x:select1[ @@ -99,7 +99,7 @@ def body_repeat_select1_itemset(r_name: str, q_name: str): @staticmethod def body_odk_rank_itemset(q_name: str): """Body has a rank with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/odk:rank[ @ref = '/test_name/{q_name}' and ./x:itemset @@ -110,7 +110,7 @@ def body_odk_rank_itemset(q_name: str): @staticmethod def body_input_label_output_value(q_name: str): """Body has an input (note) with output reference in the label.""" - return fr""" + return rf""" /h:html/h:body/x:input[@ref='/test_name/{q_name}']/x:label/x:output/@value """ From e60e7a0730219bf6fb1ccb51341cfc6388fd9467 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Mon, 15 Jan 2024 22:46:21 +1100 Subject: [PATCH 02/22] dev: remove unused and unmaintained sphinx dev docs stub - nobody has changed these files since 2011, and they're very incomplete - the docs generated aren't published anywhere - the readme instructions don't work with python 3.8 - sphinx 1.0.7 not compatible with python 3+ - newer sphinx versions have an OS dependency on OpenSSL 1.1.1+ - updated readme to refer to useful resources and current docs state --- README.rst | 9 +- docs/Makefile | 130 ------------------------ docs/source/conf.py | 226 ------------------------------------------ docs/source/index.rst | 54 ---------- 4 files changed, 5 insertions(+), 414 deletions(-) delete mode 100644 docs/Makefile delete mode 100644 docs/source/conf.py delete mode 100644 docs/source/index.rst diff --git a/README.rst b/README.rst index ef9b4641b..d949dbb5c 100644 --- a/README.rst +++ b/README.rst @@ -106,11 +106,12 @@ When writing new ``PyxformTestCase`` tests that make content assertions, it is s Documentation ============= -To check out the documentation for pyxform do the following:: +For developers, ``pyxform`` uses docstrings, type annotations, and test cases. Most modern IDEs can display docstrings and type annotations in a easily navigable format, so no additional docs are compiled (e.g. sphinx). In addition to the user documentation, developers should be familiar with the `ODK XForms Specification https://getodk.github.io/xforms-spec/`. - pip install Sphinx==1.0.7 - cd your-virtual-env-dir/src/pyxform/docs - make html +For users, ``pyxform`` has documentation at the following locations: +* `XLSForm docs https://xlsform.org/` +* `XLSForm template https://docs.google.com/spreadsheets/d/1v9Bumt3R0vCOGEKQI6ExUf2-8T72-XXp_CbKKTACuko/edit#gid=1052905058` +* `ODK Docs https://docs.getodk.org/` Change Log ========== diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 4967b0827..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,130 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pyxform.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pyxform.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/pyxform" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pyxform" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - make -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index ecd6adc00..000000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,226 +0,0 @@ -# -*- coding: utf-8 -*- -# -# pyxform documentation build configuration file, created by -# sphinx-quickstart on Thu Apr 21 15:01:53 2011. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.autodoc"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = u"pyxform" -copyright = u"2011, Columbia University, Modi Research Group" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = "0.5" -# The full version, including alpha/beta/rc tags. -release = "0.5" - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "default" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = "pyxformdoc" - - -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ( - "index", - "pyxform.tex", - u"pyxform Documentation", - u"Columbia University, Modi Research Group", - "manual", - ), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ( - "index", - "pyxform", - u"pyxform Documentation", - [u"Columbia University, Modi Research Group"], - 1, - ) -] diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 563c03b3d..000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,54 +0,0 @@ -pyxform -======= - -.. automodule:: pyxform - -xls2xform ---------- - -.. automodule:: pyxform.xls2xform - -API ---- - -pyxform takes an object oriented approach to defining -surveys. Questions, question options, groups of questions, and surveys -are all instances of SurveyElement. This allows us to model a survey -as a tree of SurveyElements. The class inheritance is structured as -follows: - -* SurveyElement - + Option - + Question - - InputQuestion - - UploadQuestion - - MultipleChoiceQuestion - + Section - - RepeatingSection - - GroupedSection - - Survey - -SurveyElement -~~~~~~~~~~~~~ - -.. autoclass:: pyxform.survey_element.SurveyElement - :members: - -Question -~~~~~~~~ - -.. autoclass:: pyxform.question.Question - :members: - -Contents -======== - -.. toctree:: - :maxdepth: 2 - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` From 3d234238b258d6b58f4e4392f3365b5c111decf1 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Mon, 15 Jan 2024 23:35:08 +1100 Subject: [PATCH 03/22] dev: fix linter warnings from flake8-bugbear rules - mostly error handling, unused variables --- pyproject.toml | 13 ++++--- pyxform/survey.py | 4 +-- pyxform/survey_element.py | 25 ++++++++------ pyxform/validators/updater.py | 4 +-- pyxform/validators/util.py | 10 +++--- pyxform/xform2json.py | 6 ++-- pyxform/xls2json.py | 36 +++++++++++--------- pyxform/xls2json_backends.py | 14 ++++---- tests/dump_and_load_tests.py | 4 +-- tests/pyxform_test_case.py | 2 +- tests/test_dynamic_default.py | 8 ++--- tests/test_external_instances_for_selects.py | 2 +- tests/test_translations.py | 8 ++--- tests/xform2json_test.py | 2 +- tests/xform_test_case/bug_tests.py | 7 ++-- 15 files changed, 77 insertions(+), 68 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2b91eadc5..d1054d39c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,14 +55,13 @@ extend-select = [ "I", # isort ] # Potentially useful rule sets to enable in future: -#select = [ -# "A", # flake8-builtins -# "B", # flake8-bugbear +select = [ + "B", # flake8-bugbear # "C4", # flake8-comprehensions -# "E", # pycodestyle error + "E", # pycodestyle error # "ERA", # eradicate (commented out code) -# "F", # pyflakes -# "I", # isort + "F", # pyflakes + "I", # isort # "PERF", # perflint # "PIE", # flake8-pie # "PL", # pylint @@ -75,7 +74,7 @@ extend-select = [ # "TRY", # tryceratops # "UP", # pyupgrade # "W", # pycodestyle warning -#] +] ignore = [ "E501", # line-too-long (we have a lot of long strings) "F821", # undefined-name (doesn't work well with type hints as of ruff 0.1.11). diff --git a/pyxform/survey.py b/pyxform/survey.py index a56aeb466..abaee3d8f 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -803,11 +803,11 @@ def _add_empty_translations(self): This disables any of the default_language fallback functionality. """ paths = {} - for lang, translation in self._translations.items(): + for translation in self._translations.values(): for path, content in translation.items(): paths[path] = paths.get(path, set()).union(content.keys()) - for lang, translation in self._translations.items(): + for lang in self._translations: for path, content_types in paths.items(): if path not in self._translations[lang]: self._translations[lang][path] = {} diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index e20238efa..f4b6ad962 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -34,6 +34,16 @@ def _overlay(over, under): return over if over else under +@lru_cache(maxsize=65536) +def any_repeat(survey_element: "SurveyElement", parent_xpath: str) -> bool: + """Return True if there ia any repeat in `parent_xpath`.""" + for item in survey_element.iter_descendants(): + if item.get_xpath() == parent_xpath and item.type == constants.REPEAT: + return True + + return False + + class SurveyElement(dict): """ SurveyElement is the base class we'll looks for the following keys @@ -41,6 +51,8 @@ class SurveyElement(dict): children, and question_type_dictionary. """ + __name__ = "SurveyElement" + # the following are important keys for the underlying dict that # describes this survey element FIELDS = { @@ -95,10 +107,6 @@ def __getattr__(self, key): def __hash__(self): return hash(id(self)) - @property - def __name__(self): - return "SurveyElement" - def __setattr__(self, key, value): self[key] = value @@ -176,14 +184,9 @@ def iter_descendants(self): for f in e.iter_descendants(): yield f - @lru_cache(maxsize=None) - def any_repeat(self, parent_xpath): + def any_repeat(self, parent_xpath: str) -> bool: """Return True if there ia any repeat in `parent_xpath`.""" - for item in self.iter_descendants(): - if item.get_xpath() == parent_xpath and item.type == constants.REPEAT: - return True - - return False + return any_repeat(survey_element=self, parent_xpath=parent_xpath) def get_lineage(self): """ diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index 4a834651b..c5303b962 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -392,8 +392,8 @@ def _install(update_info, file_name): current_mode = os.stat(new_bin_file_path).st_mode os.chmod(new_bin_file_path, current_mode | S_IXUSR | S_IXGRP) - except PyXFormError as e: - raise PyXFormError("\n\nUpdate failed!\n\n" + str(e)) + except PyXFormError as px_err: + raise PyXFormError("\n\nUpdate failed!\n\n" + str(e)) from px_err else: return latest diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index 880e55d32..e9dee5a63 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -101,7 +101,7 @@ def decode_stream(stream): return stream.decode("latin-1") except BaseException as be: msg = "Failed to decode validate stderr as utf-8 or latin-1." - raise IOError(msg, ude, be) + raise IOError(msg, ude, be) from be def request_get(url): @@ -117,16 +117,16 @@ def request_get(url): raise PyXFormError("Empty response from URL: '{u}'.".format(u=url)) else: return content - except HTTPError as e: + except HTTPError as http_err: raise PyXFormError( "Unable to fulfill request. Error code: '{c}'. " "Reason: '{r}'. URL: '{u}'." "".format(r=e.reason, c=e.code, u=url) - ) - except URLError as e: + ) from http_err + except URLError as url_err: raise PyXFormError( "Unable to reach a server. Reason: {r}. " "URL: {u}".format(r=e.reason, u=url) - ) + ) from url_err class CapturingHandler(logging.Handler): diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index f0b5e8aac..1c966d70a 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -385,8 +385,10 @@ def _get_question_from_object(self, obj, type=None): except KeyError: try: ref = obj["nodeset"] - except KeyError: - raise TypeError('cannot find "ref" or "nodeset" in {}'.format(repr(obj))) + except KeyError as node_err: + raise PyXFormError( + 'Cannot find "ref" or "nodeset" in {}'.format(repr(obj)) + ) from node_err question = { "ref": ref, "__order": self._get_question_order(ref), diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index c15f915d1..4310a0808 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -317,10 +317,10 @@ def process_range_question_type( try: has_float = any([float(x) and "." in str(x) for x in parameters.values()]) - except ValueError: + except ValueError as range_err: raise PyXFormError( "Range parameters 'start', " "'end' or 'step' must all be numbers." - ) + ) from range_err else: # is integer by default, convert to decimal if it has any float values if has_float: @@ -590,7 +590,7 @@ def workbook_to_json( list_name_choices = [option.get("name") for option in options] if len(list_name_choices) != len(set(list_name_choices)): duplicate_setting = settings.get("allow_choice_duplicates") - for k, v in Counter(list_name_choices).items(): + for v in Counter(list_name_choices).values(): if v > 1: if not duplicate_setting or duplicate_setting.capitalize() != "Yes": choice_duplicates = [ @@ -841,12 +841,12 @@ def workbook_to_json( try: int(parameters[constants.LOCATION_MIN_INTERVAL]) - except ValueError: + except ValueError as lmi_err: raise PyXFormError( "Parameter " + constants.LOCATION_MIN_INTERVAL + " must have an integer value." - ) + ) from lmi_err if int(parameters[constants.LOCATION_MIN_INTERVAL]) < 0: raise PyXFormError( "Parameter " @@ -856,12 +856,12 @@ def workbook_to_json( try: int(parameters[constants.LOCATION_MAX_AGE]) - except ValueError: + except ValueError as lma_err: raise PyXFormError( "Parameter " + constants.LOCATION_MAX_AGE + " must have an integer value." - ) + ) from lma_err if int(parameters[constants.LOCATION_MAX_AGE]) < 0: raise PyXFormError( "Parameter " @@ -1236,10 +1236,10 @@ def workbook_to_json( if not parameters["seed"].startswith("${"): try: float(parameters["seed"]) - except ValueError: + except ValueError as seed_err: raise PyXFormError( "seed value must be a number or a reference to another field." - ) + ) from seed_err elif "seed" in parameters.keys(): raise PyXFormError( "Parameters must include randomize=true to use a seed." @@ -1350,11 +1350,11 @@ def workbook_to_json( if "rows" in parameters.keys(): try: int(parameters["rows"]) - except ValueError: + except ValueError as rows_err: raise PyXFormError( (ROW_FORMAT_STRING % row_number) + " Parameter rows must have an integer value." - ) + ) from rows_err new_dict["control"] = new_dict.get("control", {}) new_dict["control"].update({"rows": parameters["rows"]}) @@ -1377,8 +1377,10 @@ def workbook_to_json( if "max-pixels" in parameters.keys(): try: int(parameters["max-pixels"]) - except ValueError: - raise PyXFormError("Parameter max-pixels must have an integer value.") + except ValueError as mp_err: + raise PyXFormError( + "Parameter max-pixels must have an integer value." + ) from mp_err new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update({"orx:max-pixels": parameters["max-pixels"]}) @@ -1474,10 +1476,10 @@ def workbook_to_json( new_dict["control"].update( {"accuracyThreshold": parameters["capture-accuracy"]} ) - except ValueError: + except ValueError as ca_err: raise PyXFormError( "Parameter capture-accuracy must have a numeric value" - ) + ) from ca_err if "warning-accuracy" in parameters.keys(): try: @@ -1485,10 +1487,10 @@ def workbook_to_json( new_dict["control"].update( {"unacceptableAccuracyThreshold": parameters["warning-accuracy"]} ) - except ValueError: + except ValueError as wa_err: raise PyXFormError( "Parameter warning-accuracy must have a numeric value" - ) + ) from wa_err parent_children_array.append(new_dict) continue diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 93155edda..4650c49cd 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -126,8 +126,8 @@ def xls_to_dict(path_or_file): workbook = xlrd.open_workbook(filename=path_or_file) else: workbook = xlrd.open_workbook(file_contents=path_or_file.read()) - except xlrd.XLRDError as error: - raise PyXFormError("Error reading .xls file: %s" % error) + except xlrd.XLRDError as read_err: + raise PyXFormError("Error reading .xls file: %s" % read_err) from read_err def xls_clean_cell(cell: xlrdCell, row_n: int, col_key: str) -> Optional[str]: value = cell.value @@ -136,8 +136,10 @@ def xls_clean_cell(cell: xlrdCell, row_n: int, col_key: str) -> Optional[str]: if not is_empty(value): try: return xls_value_to_unicode(value, cell.ctype, workbook.datemode) - except XLDateAmbiguous: - raise PyXFormError(XL_DATE_AMBIGOUS_MSG % (wb_sheet.name, col_key, row_n)) + except XLDateAmbiguous as date_err: + raise PyXFormError( + XL_DATE_AMBIGOUS_MSG % (wb_sheet.name, col_key, row_n) + ) from date_err return None @@ -217,8 +219,8 @@ def xlsx_to_dict(path_or_file): """ try: workbook = openpyxl.open(filename=path_or_file, read_only=True, data_only=True) - except (OSError, BadZipFile, KeyError) as error: - raise PyXFormError("Error reading .xlsx file: %s" % error) + except (OSError, BadZipFile, KeyError) as read_err: + raise PyXFormError("Error reading .xlsx file: %s" % read_err) from read_err def xlsx_clean_cell(cell: pyxlCell, row_n: int, col_key: str) -> Optional[str]: value = cell.value diff --git a/tests/dump_and_load_tests.py b/tests/dump_and_load_tests.py index 2ec047503..20def15e6 100644 --- a/tests/dump_and_load_tests.py +++ b/tests/dump_and_load_tests.py @@ -33,13 +33,13 @@ def setUp(self): self.surveys[filename] = create_survey_from_path(path) def test_load_from_dump(self): - for filename, survey in self.surveys.items(): + for _, survey in self.surveys.items(): survey.json_dump() path = survey.name + ".json" survey_from_dump = create_survey_from_path(path) self.assertEqual(survey.to_json_dict(), survey_from_dump.to_json_dict()) def tearDown(self): - for filename, survey in self.surveys.items(): + for _, survey in self.surveys.items(): path = survey.name + ".json" os.remove(path) diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 22d2f2dbf..a4c7880a8 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -325,7 +325,7 @@ def _pull_xml_node_from_root(element_selector): raise PyxformTestError( "ODK Validate error was thrown but 'odk_validate_error__contains' " "was empty: " + str(e) - ) + ) from e for v_err in odk_validate_error__contains: self.assertContains( e.args[0], v_err, msg_prefix="odk_validate_error__contains" diff --git a/tests/test_dynamic_default.py b/tests/test_dynamic_default.py index 268f55360..95309ef74 100644 --- a/tests/test_dynamic_default.py +++ b/tests/test_dynamic_default.py @@ -807,20 +807,20 @@ def test_dynamic_default_performance__time(self): questions = "\n".join((question.format(i=i) for i in range(1, count))) md = "".join((survey_header, questions)) - def run(name): + def run(name, case): runs = 0 results = [] while runs < 10: start = perf_counter() - self.assertPyxformXform(md=md) + self.assertPyxformXform(md=case) results.append(perf_counter() - start) runs += 1 print(name, round(sum(results) / len(results), 4)) - run(name=f"questions={count}, with check (seconds):") + run(name=f"questions={count}, with check (seconds):", case=md) with patch("pyxform.utils.default_is_dynamic", return_value=True): - run(name=f"questions={count}, without check (seconds):") + run(name=f"questions={count}, without check (seconds):", case=md) def test_dynamic_default_performance__memory(self): """ diff --git a/tests/test_external_instances_for_selects.py b/tests/test_external_instances_for_selects.py index 1d39bc9a4..048b50318 100644 --- a/tests/test_external_instances_for_selects.py +++ b/tests/test_external_instances_for_selects.py @@ -253,7 +253,7 @@ def test_param_value_case_preserved(self): | | type | name | label | parameters | | | select_one_from_file cities{ext} | city | City | value=VAL, label=lBl | """ - for ext, xp_city, xp_subs in self.xp_test_args: + for ext, xp_city, _ in self.xp_test_args: with self.subTest(msg=ext): self.assertPyxformXform( name="test", diff --git a/tests/test_translations.py b/tests/test_translations.py index 8ab87a9ea..4ad21a495 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -398,23 +398,23 @@ def test_missing_translations_check_performance(self): choice_lists = "\n".join((choice_list.format(i=i) for i in range(1, count))) md = "".join((survey_header, questions, choices_header, choice_lists)) - def run(name): + def run(name, case): runs = 0 results = [] while runs < 10: start = perf_counter() - self.assertPyxformXform(md=md) + self.assertPyxformXform(md=case) results.append(perf_counter() - start) runs += 1 print(name, sum(results) / len(results)) - run(name=f"questions={count}, with check (seconds):") + run(name=f"questions={count}, with check (seconds):", case=md) with patch( "pyxform.xls2json.SheetTranslations.missing_check", return_value=[], ): - run(name=f"questions={count}, without check (seconds):") + run(name=f"questions={count}, without check (seconds):", case=md) def test_translation_detection__survey_and_choices_columns_present(self): """Should identify that the survey is multi-language when first row(s) empty.""" diff --git a/tests/xform2json_test.py b/tests/xform2json_test.py index 90210fa04..81b1a105c 100644 --- a/tests/xform2json_test.py +++ b/tests/xform2json_test.py @@ -52,7 +52,7 @@ def test_load_from_dump(self): self.assertXFormEqual(expected, observed) def tearDown(self): - for filename, survey in self.surveys.items(): + for _, survey in self.surveys.items(): path = survey.name + ".json" if os.path.exists(path): os.remove(path) diff --git a/tests/xform_test_case/bug_tests.py b/tests/xform_test_case/bug_tests.py index 7f38f709d..d2d636efa 100644 --- a/tests/xform_test_case/bug_tests.py +++ b/tests/xform_test_case/bug_tests.py @@ -7,6 +7,7 @@ import unittest import pyxform +from pyxform.errors import PyXFormError from pyxform.utils import has_external_choices from pyxform.validators.odk_validate import ODKValidateError, check_xform from pyxform.xls2json import SurveyReader, parse_file_to_workbook_dict @@ -27,7 +28,7 @@ def runTest(self): output_path = os.path.join(test_output.PATH, root_filename + ".xml") # Do the conversion: warnings = [] - with self.assertRaises(Exception): + with self.assertRaises(PyXFormError): json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, default_name="group_name_test", warnings=warnings ) @@ -46,7 +47,7 @@ def runTest(self): output_path = os.path.join(test_output.PATH, root_filename + ".xml") # Do the conversion: warnings = [] - with self.assertRaises(Exception): + with self.assertRaises(PyXFormError): json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, default_name="not_closed_group_test", @@ -67,7 +68,7 @@ def runTest(self): output_path = os.path.join(test_output.PATH, root_filename + ".xml") # Do the conversion: warnings = [] - with self.assertRaises(Exception): + with self.assertRaises(PyXFormError): json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, default_name="duplicate_columns", warnings=warnings ) From 618abc1c136a006cd8e9b1c6581d270a5541a481 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 00:36:14 +1100 Subject: [PATCH 04/22] dev: fix linter warnings from ruff-specific rules - mostly mutable classvars, unnecessary noqa comments, list unpacking --- pyproject.toml | 3 +- pyxform/builder.py | 47 ++++++------ pyxform/constants.py | 28 ++++++-- pyxform/survey.py | 2 +- pyxform/survey_element.py | 107 +++++++++++----------------- pyxform/utils.py | 4 +- pyxform/xform2json.py | 30 ++++---- pyxform/xform_instance_parser.py | 2 +- pyxform/xls2json.py | 8 +-- pyxform/xls2json_backends.py | 4 +- tests/test_bug_round_calculation.py | 2 +- tests/test_external_instances.py | 38 +++++----- tests/test_fields.py | 6 +- tests/test_guidance_hint.py | 12 ++-- tests/test_pyxform_test_case.py | 32 +++++---- tests/test_range.py | 2 +- tests/test_repeat.py | 82 ++++++++++----------- tests/test_translations.py | 11 +-- tests/test_upload_question.py | 3 +- tests/test_validator_update.py | 10 +-- tests/test_xlsform_spec.py | 2 +- tests/xls2xform_tests.py | 22 +----- 22 files changed, 220 insertions(+), 237 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1054d39c..0a432d00d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ select = [ # "PTH", # flake8-use-pathlib # "PYI", # flake8-pyi # "RET", # flake8-return -# "RUF", # ruff-specific rules + "RUF", # ruff-specific rules # "S", # flake8-bandit # "SIM", # flake8-simplify # "TRY", # tryceratops @@ -80,3 +80,4 @@ ignore = [ "F821", # undefined-name (doesn't work well with type hints as of ruff 0.1.11). "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) ] +# per-file-ignores = {"tests/*" = ["E501"]} diff --git a/pyxform/builder.py b/pyxform/builder.py index 285c0c8e3..c67b8fc63 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -33,6 +33,23 @@ const.NAME: "other", const.LABEL: "Other", } +QUESTION_CLASSES = { + "": Question, + "action": Question, + "input": InputQuestion, + "odk:rank": MultipleChoiceQuestion, + "osm": OsmUploadQuestion, + "range": RangeQuestion, + "select": MultipleChoiceQuestion, + "select1": MultipleChoiceQuestion, + "trigger": TriggerQuestion, + "upload": UploadQuestion, +} +SECTION_CLASSES = { + const.GROUP: GroupedSection, + const.REPEAT: RepeatingSection, + const.SURVEY: Survey, +} def copy_json_dict(json_dict): @@ -60,24 +77,6 @@ def copy_json_dict(json_dict): class SurveyElementBuilder: # we use this CLASSES dict to create questions from dictionaries - QUESTION_CLASSES = { - "": Question, - "action": Question, - "input": InputQuestion, - "odk:rank": MultipleChoiceQuestion, - "osm": OsmUploadQuestion, - "range": RangeQuestion, - "select": MultipleChoiceQuestion, - "select1": MultipleChoiceQuestion, - "trigger": TriggerQuestion, - "upload": UploadQuestion, - } - - SECTION_CLASSES = { - const.GROUP: GroupedSection, - const.REPEAT: RepeatingSection, - const.SURVEY: Survey, - } def __init__(self, **kwargs): # I don't know why we would need an explicit none option for @@ -111,7 +110,7 @@ def create_survey_element_from_dict( if "add_none_option" in d: self._add_none_option = d["add_none_option"] - if d[const.TYPE] in self.SECTION_CLASSES: + if d[const.TYPE] in SECTION_CLASSES: if d[const.TYPE] == const.SURVEY: self._choices = copy.deepcopy(d.get(const.CHOICES, {})) @@ -257,7 +256,7 @@ def _get_question_class(question_type_str, question_type_dictionary): if control_tag == "upload" and control_dict.get("mediatype") == "osm/*": control_tag = "osm" - return SurveyElementBuilder.QUESTION_CLASSES[control_tag] + return QUESTION_CLASSES[control_tag] @staticmethod def _create_specify_other_question_from_dict(d: Dict[str, Any]) -> InputQuestion: @@ -272,7 +271,7 @@ def _create_specify_other_question_from_dict(d: Dict[str, Any]) -> InputQuestion def _create_section_from_dict(self, d): d_copy = d.copy() children = d_copy.pop(const.CHILDREN, []) - section_class = self.SECTION_CLASSES[d_copy[const.TYPE]] + section_class = SECTION_CLASSES[d_copy[const.TYPE]] if d[const.TYPE] == const.SURVEY and const.TITLE not in d: d_copy[const.TITLE] = d[const.NAME] result = section_class(**d_copy) @@ -394,9 +393,9 @@ def create_survey_from_xls(path_or_file, default_name=None): def create_survey( - name_of_main_section: str = None, - sections: Dict[str, Dict] = None, - main_section: Dict[str, Any] = None, + name_of_main_section: Optional[str] = None, + sections: Optional[Dict[str, Dict]] = None, + main_section: Optional[Dict[str, Any]] = None, id_string: Optional[str] = None, title: Optional[str] = None, ) -> Survey: diff --git a/pyxform/constants.py b/pyxform/constants.py index 792072c10..61c7684fe 100644 --- a/pyxform/constants.py +++ b/pyxform/constants.py @@ -42,9 +42,7 @@ ATTRIBUTE = "attribute" ALLOW_CHOICE_DUPLICATES = "allow_choice_duplicates" -BIND = ( - "bind" # TODO: What should I do with the nested types? (readonly and relevant) # noqa -) +BIND = "bind" # TODO: What should I do with the nested types? (readonly and relevant) MEDIA = "media" CONTROL = "control" APPEARANCE = "appearance" @@ -72,7 +70,7 @@ # XLS Specific constants LIST_NAME = "list name" CASCADING_SELECT = "cascading_select" -TABLE_LIST = "table-list" # hyphenated because it goes in appearance, and convention for appearance column is dashes # noqa +TABLE_LIST = "table-list" # hyphenated because it goes in appearance, and convention for appearance column is dashes FIELD_LIST = "field-list" LIST_NOLABEL = "list-nolabel" @@ -150,3 +148,25 @@ class EntityColumns(StrEnum): "prefix the sheet name with an underscore. For example 'setting' " "becomes '_setting'." ) + +BINDING_CONVERSIONS = { + "yes": "true()", + "Yes": "true()", + "YES": "true()", + "true": "true()", + "True": "true()", + "TRUE": "true()", + "no": "false()", + "No": "false()", + "NO": "false()", + "false": "false()", + "False": "false()", + "FALSE": "false()", +} +CONVERTIBLE_BIND_ATTRIBUTES = ( + "readonly", + "required", + "relevant", + "constraint", + "calculate", +) diff --git a/pyxform/survey.py b/pyxform/survey.py index abaee3d8f..2fef5e788 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -708,7 +708,7 @@ def _setup_translations(self): def _setup_choice_translations( name, choice_value, itext_id ) -> Generator[Tuple[List[str], str], None, None]: - for media_or_lang, value in choice_value.items(): # noqa + for media_or_lang, value in choice_value.items(): if isinstance(value, dict): for language, val in value.items(): yield ([language, itext_id, media_or_lang], val) diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index f4b6ad962..c16222107 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -6,9 +6,9 @@ import re from collections import deque from functools import lru_cache -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List -from pyxform import constants +from pyxform import constants as const from pyxform.errors import PyXFormError from pyxform.question_type_dictionary import QUESTION_TYPE_DICT from pyxform.utils import ( @@ -21,10 +21,40 @@ from pyxform.xlsparseutils import is_valid_xml_tag if TYPE_CHECKING: - from typing import List - from pyxform.utils import DetachableElement +# The following are important keys for the underlying dict that describes SurveyElement +FIELDS = { + "name": str, + const.COMPACT_TAG: str, # used for compact (sms) representation + "sms_field": str, + "sms_option": str, + "label": str, + "hint": str, + "guidance_hint": str, + "default": str, + "type": str, + "appearance": str, + "parameters": dict, + "intent": str, + "jr:count": str, + "bind": dict, + "instance": dict, + "control": dict, + "media": dict, + # this node will also have a parent and children, like a tree! + "parent": lambda: None, + "children": list, + "itemset": str, + "choice_filter": str, + "query": str, + "autoplay": str, + "flat": lambda: False, + "action": str, + "list_name": str, + "trigger": str, +} + def _overlay(over, under): if isinstance(under, dict): @@ -38,7 +68,7 @@ def _overlay(over, under): def any_repeat(survey_element: "SurveyElement", parent_xpath: str) -> bool: """Return True if there ia any repeat in `parent_xpath`.""" for item in survey_element.iter_descendants(): - if item.get_xpath() == parent_xpath and item.type == constants.REPEAT: + if item.get_xpath() == parent_xpath and item.type == const.REPEAT: return True return False @@ -52,39 +82,7 @@ class SurveyElement(dict): """ __name__ = "SurveyElement" - - # the following are important keys for the underlying dict that - # describes this survey element - FIELDS = { - "name": str, - constants.COMPACT_TAG: str, # used for compact (sms) representation - "sms_field": str, - "sms_option": str, - "label": str, - "hint": str, - "guidance_hint": str, - "default": str, - "type": str, - "appearance": str, - "parameters": dict, - "intent": str, - "jr:count": str, - "bind": dict, - "instance": dict, - "control": dict, - "media": dict, - # this node will also have a parent and children, like a tree! - "parent": lambda: None, - "children": list, - "itemset": str, - "choice_filter": str, - "query": str, - "autoplay": str, - "flat": lambda: False, - "action": str, - "list_name": str, - "trigger": str, - } + FIELDS: ClassVar[Dict[str, Any]] = FIELDS.copy() def _default(self): # TODO: need way to override question type dictionary @@ -137,29 +135,6 @@ def add_children(self, children): else: self.add_child(children) - BINDING_CONVERSIONS = { - "yes": "true()", - "Yes": "true()", - "YES": "true()", - "true": "true()", - "True": "true()", - "TRUE": "true()", - "no": "false()", - "No": "false()", - "NO": "false()", - "false": "false()", - "False": "false()", - "FALSE": "false()", - } - - CONVERTIBLE_BIND_ATTRIBUTES = ( - "readonly", - "required", - "relevant", - "constraint", - "calculate", - ) - # Supported media types for attaching to questions SUPPORTED_MEDIA = ("image", "big-image", "audio", "video") @@ -167,7 +142,7 @@ def validate(self): if not is_valid_xml_tag(self.name): invalid_char = re.search(INVALID_XFORM_TAG_REGEXP, self.name) raise PyXFormError( - f"The name '{self.name}' contains an invalid character '{invalid_char.group(0)}'. Names {constants.XML_IDENTIFIER_ERROR_MESSAGE}" + f"The name '{self.name}' contains an invalid character '{invalid_char.group(0)}'. Names {const.XML_IDENTIFIER_ERROR_MESSAGE}" ) # TODO: Make sure renaming this doesn't cause any problems @@ -419,7 +394,7 @@ def xml_hint(self): hint, output_inserted = self.get_root().insert_output_values(self.hint, self) return node("hint", hint, toParseString=output_inserted) - def xml_label_and_hint(self) -> "List[DetachableElement]": + def xml_label_and_hint(self) -> List["DetachableElement"]: """ Return a list containing one node for the label and if there is a hint one node for the hint. @@ -470,10 +445,10 @@ def xml_bindings(self): # the xls2json side. if ( hashable(v) - and v in self.BINDING_CONVERSIONS - and k in self.CONVERTIBLE_BIND_ATTRIBUTES + and v in const.BINDING_CONVERSIONS + and k in const.CONVERTIBLE_BIND_ATTRIBUTES ): - v = self.BINDING_CONVERSIONS[v] + v = const.BINDING_CONVERSIONS[v] if k == "jr:constraintMsg" and ( isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) ): diff --git a/pyxform/utils.py b/pyxform/utils.py index 968f2d062..628bb6652 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -375,7 +375,7 @@ def levenshtein_distance(a: str, b: str) -> int: return v0[n] -def get_expression_lexer() -> re.Scanner: # noqa +def get_expression_lexer() -> re.Scanner: """ Get a expression lexer (scanner) for parsing. """ @@ -439,7 +439,7 @@ def tokenizer(scan, value): lexicon = [(v, get_tokenizer(k)) for k, v in lexer_rules.items()] # re.Scanner is undocumented but has been around since at least 2003 # https://mail.python.org/pipermail/python-dev/2003-April/035075.html - return re.Scanner(lexicon) # noqa + return re.Scanner(lexicon) # Scanner takes a few 100ms to compile so use this shared instance. diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 1c966d70a..02e8ad319 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -20,6 +20,15 @@ logger.addHandler(logging.NullHandler()) +QUESTION_TYPES = { + "select": "select all that apply", + "select1": "select one", + "int": "integer", + "dateTime": "datetime", + "string": "text", +} + + # {{{ http://code.activestate.com/recipes/573463/ (r7) class XmlDictObject(dict): """ @@ -202,14 +211,6 @@ def create_survey_element_from_xml(xml_file): class XFormToDictBuilder: """Experimental XFORM xml to XFORM JSON""" - QUESTION_TYPES = { - "select": "select all that apply", - "select1": "select one", - "int": "integer", - "dateTime": "datetime", - "string": "text", - } - def __init__(self, xml_file): doc_as_dict = XFormToDict(xml_file).get_dict() self._xmldict = doc_as_dict @@ -461,7 +462,7 @@ def _get_question_from_object(self, obj, type=None): "select1", "select", ]: # Select bind type is 'string' https://github.com/XLSForm/pyxform/issues/168 - question["type"] = self.QUESTION_TYPES[type] + question["type"] = QUESTION_TYPES[type] if question_type == "geopoint" and "hint" in question: del question["hint"] if "type" not in question and type: @@ -507,7 +508,7 @@ def _get_question_params_from_bindings(self, ref): if k == "nodeset": continue if k == "type": - v = self._get_question_type(v) + v = self._get_question_type(question_type=v) if k in [ "relevant", "required", @@ -548,10 +549,11 @@ def _get_question_params_from_bindings(self, ref): return rs return None - def _get_question_type(self, type): - if type in self.QUESTION_TYPES.keys(): - return self.QUESTION_TYPES[type] - return type + @staticmethod + def _get_question_type(question_type): + if question_type in QUESTION_TYPES.keys(): + return QUESTION_TYPES[question_type] + return question_type def _get_translations(self) -> List[Dict]: if "itext" not in self.model: diff --git a/pyxform/xform_instance_parser.py b/pyxform/xform_instance_parser.py index 93d4a9afc..e16d257f3 100644 --- a/pyxform/xform_instance_parser.py +++ b/pyxform/xform_instance_parser.py @@ -46,7 +46,7 @@ def _flatten_dict(d, prefix): assert isinstance(prefix, list) for key, value in d.items(): - new_prefix = prefix + [key] + new_prefix = [*prefix, key] if isinstance(value, dict): for pair in _flatten_dict(value, new_prefix): yield pair diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 4310a0808..a07e8bc31 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -564,7 +564,7 @@ def workbook_to_json( "On the choices sheet there is a option with no label. " + info ) # chrislrobert's fix for a cryptic error message: - # see: https://code.google.com/p/opendatakit/issues/detail?id=832&start=200 # noqa + # see: https://code.google.com/p/opendatakit/issues/detail?id=832&start=200 option_keys = list(option.keys()) for headername in option_keys: # Using warnings and removing the bad columns @@ -613,7 +613,7 @@ def workbook_to_json( ] ), ) - ) # noqa + ) # ########## Entities sheet ########### entities_sheet = workbook_dict.get(constants.ENTITIES, []) @@ -1294,7 +1294,7 @@ def workbook_to_json( constants.TYPE: select_type, constants.NAME: "reserved_name_for_field_list_labels_" + str(row_number), - # Adding row number for uniqueness # noqa + # Adding row number for uniqueness constants.CONTROL: {constants.APPEARANCE: "label"}, constants.CHOICES: choices[list_name], constants.ITEMSET: list_name, @@ -1511,7 +1511,7 @@ def workbook_to_json( # print "Generating flattened instance..." add_flat_annotations(stack[0]["parent_children"]) - meta_children = [] + survey_meta + meta_children = [*survey_meta] if aliases.yes_no.get(settings.get("omit_instanceID")): if settings.get("public_key"): diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 4650c49cd..e3960d832 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -423,7 +423,7 @@ def convert_file_to_csv_string(path): for out_key in out_keys: out_row.append(row.get(out_key, None)) out_rows.append(out_row) - writer.writerow([None] + out_keys) + writer.writerow([None, *out_keys]) for out_row in out_rows: - writer.writerow([None] + out_row) + writer.writerow([None, *out_row]) return foo.getvalue() diff --git a/tests/test_bug_round_calculation.py b/tests/test_bug_round_calculation.py index 8627ce333..ffcf6418c 100644 --- a/tests/test_bug_round_calculation.py +++ b/tests/test_bug_round_calculation.py @@ -14,6 +14,6 @@ def test_non_existent_itext_reference(self): | | type | name | label | calculation | | | decimal | amount | Counter | | | | calculate | rounded | Rounded | round(${amount}, 0) | - """, # noqa + """, xml__contains=[""""""], ) diff --git a/tests/test_external_instances.py b/tests/test_external_instances.py index be23a67ce..b6f89b776 100644 --- a/tests/test_external_instances.py +++ b/tests/test_external_instances.py @@ -172,7 +172,7 @@ def test_cannot__use_same_external_xml_id_with_mixed_types(self): | | begin group | g4 | | | | | calculate | city | City | pulldata('fruits', 'name', 'name', 'mango') | | | end group | g4 | | | - """, # noqa + """, model__contains=[], ) self.assertIn("The name 'city' was found 2 time(s)", repr(ctx.exception)) @@ -197,7 +197,7 @@ def test_can__use_same_external_csv_id_with_mixed_types(self): | | text | foo | Foo | | | | calculate | city | City | pulldata('fruits', 'name', 'name', 'mango') | | | end group | g4 | | | - """, # noqa + """, model__contains=[''], ) @@ -226,12 +226,12 @@ def test_can__use_all_types_together_with_unique_ids(self): | | list_name | name | label | | | | | states | 1 | Pass | | | | | states | 2 | Fail | | | - """, # noqa + """, model__contains=[ '', '', '', - ], # noqa + ], xml__xpath_match=[ xpc.model_instance_choices_label("states", (("1", "Pass"), ("2", "Fail"))) ], @@ -248,7 +248,7 @@ def test_cannot__use_different_src_same_id__select_then_internal(self): | | list_name | name | label | | | | states | 1 | Pass | | | | states | 2 | Fail | | - """ # noqa + """ with self.assertRaises(PyXFormError) as ctx: survey = self.md_to_pyxform_survey(md_raw=md) survey._to_pretty_xml() @@ -272,14 +272,14 @@ def test_cannot__use_different_src_same_id__external_then_pulldata(self): | | calculate | f_csv | City | pulldata('fruits', 'name', 'name', 'mango') | | | note | note | Fruity! ${f_csv} | | | | end group | g1 | | | - """ # noqa + """ with self.assertRaises(PyXFormError) as ctx: survey = self.md_to_pyxform_survey(md_raw=md) survey._to_pretty_xml() self.assertIn( "Instance name: 'fruits', " "Existing type: 'external', Existing URI: 'jr://file/fruits.xml', " - "Duplicate type: 'pulldata', Duplicate URI: 'jr://file-csv/fruits.csv', " # noqa + "Duplicate type: 'pulldata', Duplicate URI: 'jr://file-csv/fruits.csv', " "Duplicate context: '[type: group, name: g1]'.", repr(ctx.exception), ) @@ -296,14 +296,14 @@ def test_cannot__use_different_src_same_id__pulldata_then_external(self): | | xml-external | fruits | | | | | note | note | Fruity! ${f_csv} | | | | end group | g1 | | | - """ # noqa + """ with self.assertRaises(PyXFormError) as ctx: survey = self.md_to_pyxform_survey(md_raw=md) survey._to_pretty_xml() self.assertIn( "Instance name: 'fruits', " - "Existing type: 'pulldata', Existing URI: 'jr://file-csv/fruits.csv', " # noqa - "Duplicate type: 'external', Duplicate URI: 'jr://file/fruits.xml', " # noqa + "Existing type: 'pulldata', Existing URI: 'jr://file-csv/fruits.csv', " + "Duplicate type: 'external', Duplicate URI: 'jr://file/fruits.xml', " "Duplicate context: '[type: group, name: g1]'.", repr(ctx.exception), ) @@ -319,10 +319,10 @@ def test_can__reuse_csv__selects_then_pulldata(self): | | select_one_from_file pain_locations.csv | pyear | Location of worst pain this year. | | | | calculate | f_csv | pd | pulldata('pain_locations', 'name', 'name', 'arm') | | | note | note | Arm ${f_csv} | | - """ # noqa + """ expected = """ -""" # noqa +""" self.assertPyxformXform(md=md, model__contains=[expected]) survey = self.md_to_pyxform_survey(md_raw=md) xml = survey._to_pretty_xml() @@ -339,9 +339,9 @@ def test_can__reuse_csv__pulldata_then_selects(self): | | select_one_from_file pain_locations.csv | pweek | Location of worst pain this week. | | | | select_one_from_file pain_locations.csv | pmonth | Location of worst pain this month. | | | | select_one_from_file pain_locations.csv | pyear | Location of worst pain this year. | | - """ # noqa + """ expected = ( - """""" # noqa + """""" ) self.assertPyxformXform(md=md, model__contains=[expected]) @@ -355,10 +355,10 @@ def test_can__reuse_xml__selects_then_external(self): | | select_one_from_file pain_locations.xml | pmonth | Location of worst pain this month. | | | select_one_from_file pain_locations.xml | pyear | Location of worst pain this year. | | | xml-external | pain_locations | | - """ # noqa + """ expected = """ -""" # noqa +""" survey = self.md_to_pyxform_survey(md_raw=md) xml = survey._to_pretty_xml() self.assertEqual(1, xml.count(expected)) @@ -373,9 +373,9 @@ def test_can__reuse_xml__external_then_selects(self): | | select_one_from_file pain_locations.xml | pweek | Location of worst pain this week. | | | select_one_from_file pain_locations.xml | pmonth | Location of worst pain this month. | | | select_one_from_file pain_locations.xml | pyear | Location of worst pain this year. | - """ # noqa + """ expected = ( - """""" # noqa + """""" ) self.assertPyxformXform(md=md, model__contains=[expected]) survey = self.md_to_pyxform_survey(md_raw=md) @@ -607,7 +607,7 @@ def test_mixed_quotes_and_functions_in_pulldata(self): | | calculate | calculate2 | | pulldata('instance2',"last",'rcid',concat('RC',${rcid})) | | | calculate | calculate3 | | pulldata('instance3','envelope','rcid',"Bar") | | | calculate | calculate4 | | pulldata('instance4' ,'envelope','rcid',"Bar") | - """, # noqa + """, xml__contains=[ '', '', diff --git a/tests/test_fields.py b/tests/test_fields.py index a1df844c1..61f001961 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -59,7 +59,7 @@ def test_duplicate_choices_without_setting(self): errored=True, error__contains=[ "The name column for the 'list' choice list contains these duplicates: 'b'" - ], # noqa + ], ) def test_multiple_duplicate_choices_without_setting(self): @@ -78,7 +78,7 @@ def test_multiple_duplicate_choices_without_setting(self): errored=True, error__contains=[ "The name column for the 'list' choice list contains these duplicates: 'a', 'b'" - ], # noqa + ], ) def test_duplicate_choices_with_setting_not_set_to_yes(self): @@ -99,7 +99,7 @@ def test_duplicate_choices_with_setting_not_set_to_yes(self): errored=True, error__contains=[ "The name column for the 'list' choice list contains these duplicates: 'b'" - ], # noqa + ], ) def test_duplicate_choices_with_allow_choice_duplicates_setting(self): diff --git a/tests/test_guidance_hint.py b/tests/test_guidance_hint.py index d292907f9..90e063cec 100644 --- a/tests/test_guidance_hint.py +++ b/tests/test_guidance_hint.py @@ -28,7 +28,7 @@ def test_guidance_hint_and_label(self): | survey | | | | | | | type | name | label | guidance_hint | | | string | name | Name | as shown on birth certificate| - """, # noqa + """, xml__contains=[ "", 'as shown on birth certificate', @@ -44,7 +44,7 @@ def test_hint_and_guidance_one_language(self): # pylint: disable=C0103 | survey | | | | | | | | type | name | label | hint | guidance_hint | | | string | name | Name | your name | as shown on birth certificate| - """, # noqa + """, xml__contains=[ "", "your name", @@ -60,10 +60,10 @@ def test_multi_language_guidance(self): | survey | | | | | | | | | type | name | label | hint | guidance_hint | guidance_hint::French (fr) | | | string | name | Name | your name | as shown on birth certificate| comme sur le certificat de naissance| - """, # noqa + """, xml__contains=[ '', - 'comme sur le certificat de naissance', # noqa + 'comme sur le certificat de naissance', '', 'as shown on birth certificate', "", @@ -91,7 +91,7 @@ def test_multi_language_guidance_only(self): # pylint:disable=C0103 | survey | | | | | | | type | name | guidance_hint | guidance_hint::French (fr) | | | string | name | as shown on birth certificate| comme sur le certificat de naissance | - """, # noqa + """, errored=True, error__contains=["The survey element named 'name' has no label or hint."], ) @@ -104,7 +104,7 @@ def test_multi_language_hint(self): | survey | | | | | | | type | name | hint | hint::French (fr) | | | string | name | default language hint| French hint | - """, # noqa + """, xml__contains=[ "", "French hint", diff --git a/tests/test_pyxform_test_case.py b/tests/test_pyxform_test_case.py index 446e56dc6..8de8fa1fe 100644 --- a/tests/test_pyxform_test_case.py +++ b/tests/test_pyxform_test_case.py @@ -79,10 +79,10 @@ class TestPyxformTestCaseXmlXpath(PyxformTestCase): count=1, ) # Convenience combinations of the above data for Suite 1 tests. - suite1 = [s1c1, s1c2, s1c3, s1c4, s1c5] - suite1_counts = [c.ctuple for c in suite1] - suite1_exacts = [c.etuple for c in suite1] - suite1_xpaths = [c.xpath for c in suite1] + suite1 = (s1c1, s1c2, s1c3, s1c4, s1c5) + suite1_counts = tuple(c.ctuple for c in suite1) + suite1_exacts = tuple(c.etuple for c in suite1) + suite1_xpaths = tuple(c.xpath for c in suite1) # Suite 2: multiple expected match results. # s2c1: element in default namespace. @@ -131,10 +131,10 @@ class TestPyxformTestCaseXmlXpath(PyxformTestCase): count=2, ) # Convenience combinations of the above data for Suite 2 tests. - suite2 = [s2c1, s2c2, s2c3] - suite2_counts = [c.ctuple for c in suite2] - suite2_exacts = [c.etuple for c in suite2] - suite2_xpaths = [c.xpath for c in suite2] + suite2 = (s2c1, s2c2, s2c3) + suite2_counts = tuple(c.ctuple for c in suite2) + suite2_exacts = tuple(c.etuple for c in suite2) + suite2_xpaths = tuple(c.xpath for c in suite2) # Suite 3: other misc cases. # s3c1: XPath with no expected matches. @@ -220,7 +220,7 @@ def test_xml__n_xpath_1_match_fail__xpath_exact(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_exact=self.suite1_exacts + [(self.s1c1.xpath, {"bananas"})], + xml__xpath_exact=[*self.suite1_exacts, (self.s1c1.xpath, {"bananas"})], run_odk_validate=False, ) @@ -229,7 +229,7 @@ def test_xml__n_xpath_1_match_fail__xpath_count(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_count=self.suite1_counts + [(self.s1c1.xpath, 5)], + xml__xpath_count=[*self.suite1_counts, (self.s1c1.xpath, 5)], run_odk_validate=False, ) @@ -238,7 +238,7 @@ def test_xml__n_xpath_1_match_fail__xpath_match(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_match=self.suite1_xpaths + [self.s3c1.xpath], + xml__xpath_match=[*self.suite1_xpaths, self.s3c1.xpath], run_odk_validate=False, ) @@ -314,8 +314,10 @@ def test_xml__n_xpath_n_match_fail__xpath_exact(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_exact=self.suite2_exacts - + [(self.s2c3.xpath, {"bananas", "eggs"})], + xml__xpath_exact=[ + *self.suite2_exacts, + (self.s2c3.xpath, {"bananas", "eggs"}), + ], run_odk_validate=False, ) @@ -324,7 +326,7 @@ def test_xml__n_xpath_n_match_fail__xpath_count(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_count=self.suite2_counts + [(self.s2c3.xpath, 5)], + xml__xpath_count=[*self.suite2_counts, (self.s2c3.xpath, 5)], run_odk_validate=False, ) @@ -333,6 +335,6 @@ def test_xml__n_xpath_n_match_fail__xpath_match(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_match=self.suite2_xpaths + [self.s1c1.xpath], + xml__xpath_match=[*self.suite2_xpaths, self.s1c1.xpath], run_odk_validate=False, ) diff --git a/tests/test_range.py b/tests/test_range.py index 85850fa02..39402d58a 100644 --- a/tests/test_range.py +++ b/tests/test_range.py @@ -168,7 +168,7 @@ def test_range_comma_separator(self): | survey | | | | | | | type | name | label | parameters | | | range | level | Scale | start = 1 , end = 10 , step = 2 | - """, # noqa + """, xml__contains=[ '', '', diff --git a/tests/test_repeat.py b/tests/test_repeat.py index 013ba2ff4..655ea474e 100644 --- a/tests/test_repeat.py +++ b/tests/test_repeat.py @@ -64,7 +64,7 @@ def test_repeat_relative_reference(self): | | note | note9 | | ${FF} ${H} ${N} ${N} | | | end repeat | | | | | | | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long instance__contains=[ '
', "", @@ -119,7 +119,7 @@ def test_calculate_relative_path(self): | | crop_list | maize | Maize | | | | crop_list | beans | Beans | | | | crop_list | kale | Kale | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long model__contains=[ """""", @@ -149,10 +149,10 @@ def test_choice_filter_relative_path(self): # pylint: disable=invalid-name | | crop_list | maize | Maize | | | | crop_list | beans | Beans | | | | crop_list | kale | Kale | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""", # noqa pylint: disable=line-too-long - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -181,9 +181,9 @@ def test_indexed_repeat_relative_path(self): | | crop_list | maize | Maize | | | | crop_list | beans | Beans | | | | crop_list | kale | Kale | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long model__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -267,7 +267,7 @@ def test_hints_are_not_present_within_repeats(self): | | pet | cat | Cat | | | | pet | bird | Bird | | | | pet | fish | Fish | | - """ # noqa + """ self.assertPyxformXform( md=md, xml__xpath_match=[ @@ -294,7 +294,7 @@ def test_hints_are_present_within_groups(self): | | text | child_name | Name of child? | Should be a text | | | decimal | birthweight | Child birthweight (in kgs)? | Should be a decimal | | | end group | | | | - """ # noqa + """ expected = """ @@ -305,7 +305,7 @@ def test_hints_are_present_within_groups(self): Should be a decimal - """ # noqa + """ self.assertPyxformXform(md=md, xml__contains=[expected]) @@ -485,9 +485,9 @@ def test_indexed_repeat_regular_calculation_relative_path_exception(self): | | text | name | Enter name | | | | text | prev_name | Name in previous repeat instance | indexed-repeat(${name}, ${person}, ${pos}-1) | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long model__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -503,9 +503,9 @@ def test_indexed_repeat_dynamic_default_relative_path_exception(self): | | text | name | Enter name | | | | text | prev_name | Name in previous repeat instance | indexed-repeat(${name}, ${person}, position(..)-1) | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -524,9 +524,9 @@ def test_indexed_repeat_nested_repeat_relative_path_exception(self): | | text | prev_name | Non-sensible previous name in first family, 2nd person | indexed-repeat(${name}, ${family}, 1, ${person}, 2) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -548,9 +548,9 @@ def test_indexed_repeat_math_expression_nested_repeat_relative_path_exception( | | text | prev_name | Expression label | 7 * indexed-repeat(${age}, ${family}, 1, ${person}, 2) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -572,9 +572,9 @@ def test_multiple_indexed_repeat_in_expression_nested_repeat_relative_path_excep | | text | prev_name | Expression label | concat(indexed-repeat(${name}, ${family}, 1, ${person}, 2), indexed-repeat(${age}, ${family}, 1, ${person}, 2)) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -596,9 +596,9 @@ def test_mixed_variables_and_indexed_repeat_in_expression_text_type_nested_repea | | text | prev_name | Expression label | concat(${name}, indexed-repeat(${age}, ${family}, 1, ${person}, 2), ${age}) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -620,9 +620,9 @@ def test_mixed_variables_and_indexed_repeat_in_expression_integer_type_nested_re | | integer | bp_dia | Diastolic pressure | if(${bp_row} = 1, '', indexed-repeat(${bp_dia}, ${bp_rg}, ${bp_row} - 1)) | | | end repeat | | | | | | end group | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -644,9 +644,9 @@ def test_indexed_repeat_math_expression_with_double_variable_in_nested_repeat_re | | text | prev_name | Expression label | ${age} > indexed-repeat(${age}, ${family}, 1, ${person}, 2) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -677,7 +677,7 @@ def test_repeat_using_select_with_reference_path_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -708,7 +708,7 @@ def test_repeat_using_select_uses_current_with_reference_path_in_predicate_and_i name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -734,7 +734,7 @@ def test_repeat_and_group_with_reference_path_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -758,7 +758,7 @@ def test_repeat_with_reference_path_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -784,9 +784,9 @@ def test_repeat_with_reference_path_with_spaces_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long - """""", # noqa pylint: disable=line-too-long - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long + """""", # pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -810,7 +810,7 @@ def test_repeat_with_reference_path_in_a_method_with_spaces_in_predicate_uses_cu name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -834,7 +834,7 @@ def test_repeat_with_reference_path_with_spaces_in_predicate_with_parenthesis_us name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -855,7 +855,7 @@ def test_relative_path_expansion_not_using_current_if_reference_path_is_predicat name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -878,7 +878,7 @@ def test_relative_path_expansion_not_using_current_if_reference_path_is_predicat name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -902,7 +902,7 @@ def test_repeat_with_reference_path_in_multiple_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -928,7 +928,7 @@ def test_repeat_with_reference_path_in_multiple_complex_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -954,7 +954,7 @@ def test_repeat_with_reference_path_after_instance_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -980,7 +980,7 @@ def test_repeat_with_reference_path_after_instance_not_in_predicate_not_using_cu name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) diff --git a/tests/test_translations.py b/tests/test_translations.py index 4ad21a495..c5b0bfd2b 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -323,8 +323,8 @@ def test_missing_translation__one_lang_all_cols(self): self.assertPyxformXform( md=md, warnings__contains=[warning], - xml__xpath_match=common_xpaths - + [ + xml__xpath_match=[ + *common_xpaths, xp.question_itext_label(DEFAULT_LANG, "hello"), xp.question_itext_hint(DEFAULT_LANG, "-"), xp.question_itext_form(DEFAULT_LANG, "guidance", "-"), @@ -361,8 +361,11 @@ def test_missing_translation__one_lang_all_cols(self): md=settings + md, # TODO: bug - missing default lang translatable/itext values. # warnings__contains=[warning], - xml__xpath_match=common_xpaths - + [xp.language_is_default("eng"), xp.language_no_itext(DEFAULT_LANG)], + xml__xpath_match=[ + *common_xpaths, + xp.language_is_default("eng"), + xp.language_no_itext(DEFAULT_LANG), + ], ) @unittest.skip("Slow performance test. Un-skip to run as needed.") diff --git a/tests/test_upload_question.py b/tests/test_upload_question.py index c6abf6d6e..c611e940a 100644 --- a/tests/test_upload_question.py +++ b/tests/test_upload_question.py @@ -73,6 +73,7 @@ def test_image_question_custom_col_calc(self): | | text | watermark_phrase | Watermark Text: | | | | text | text1 | Text | | | | image | image1 | Take a Photo: | watermark=${watermark_phrase} | - """, # noqa + """, + errored=False, xml__contains=["watermark= /data/watermark_phrase "], ) diff --git a/tests/test_validator_update.py b/tests/test_validator_update.py index 735c1b533..4deb2f88e 100644 --- a/tests/test_validator_update.py +++ b/tests/test_validator_update.py @@ -363,7 +363,7 @@ def test_unzip_find_zip_jobs__ok_real_current(self): open_zip_file=zip_file, bin_paths=bin_paths, out_path=temp_dir ) self.assertEqual(3, len(jobs.keys())) - self.assertTrue(list(jobs.keys())[0].startswith(temp_dir)) + self.assertTrue(next(iter(jobs.keys())).startswith(temp_dir)) def test_unzip_find_zip_jobs__ok_real_ideal(self): """Should return a list of zip jobs same length as search.""" @@ -377,7 +377,7 @@ def test_unzip_find_zip_jobs__ok_real_ideal(self): open_zip_file=zip_file, bin_paths=bin_paths, out_path=temp_dir ) self.assertEqual(3, len(jobs.keys())) - self.assertTrue(list(jobs.keys())[0].startswith(temp_dir)) + self.assertTrue(next(iter(jobs.keys())).startswith(temp_dir)) def test_unzip_find_zip_jobs__ok_real_dupes(self): """Should return a list of zip jobs same length as search.""" @@ -391,7 +391,7 @@ def test_unzip_find_zip_jobs__ok_real_dupes(self): open_zip_file=zip_file, bin_paths=bin_paths, out_path=temp_dir ) self.assertEqual(3, len(jobs.keys())) - self.assertTrue(list(jobs.keys())[0].startswith(temp_dir)) + self.assertTrue(next(iter(jobs.keys())).startswith(temp_dir)) def test_unzip_find_zip_jobs__not_found_raises(self): """Should raise an error if zip jobs isn't same length as search.""" @@ -420,9 +420,9 @@ def test_unzip_extract_file__bad_crc_raises(self): with get_temp_dir() as temp_dir, ZipFile( self.zip_file, mode="r" ) as zip_file, self.assertRaises(BadZipFile) as ctx: - zip_item = [ + zip_item = next( x for x in zip_file.infolist() if x.filename.endswith("validate") - ][0] + ) zip_item.CRC = 12345 file_out_path = os.path.join(temp_dir, "validate") self.updater._unzip_extract_file( diff --git a/tests/test_xlsform_spec.py b/tests/test_xlsform_spec.py index 42e72f281..4f76f2880 100644 --- a/tests/test_xlsform_spec.py +++ b/tests/test_xlsform_spec.py @@ -53,7 +53,7 @@ def test_warnings__count(self): | settings | | | | | | | | form_title | form_id | public_key | submission_url | default_language | | | spec_test | spec_test | | | | - """ # noqa + """ warnings = [] self.assertPyxformXform( debug=True, diff --git a/tests/xls2xform_tests.py b/tests/xls2xform_tests.py index c7d8119e6..4a65f7f60 100644 --- a/tests/xls2xform_tests.py +++ b/tests/xls2xform_tests.py @@ -8,9 +8,8 @@ import argparse import logging -from unittest import TestCase +from unittest import TestCase, mock -import pyxform from pyxform.xls2xform import ( _create_parser, _validator_args_logic, @@ -20,27 +19,8 @@ from tests.utils import path_to_text_fixture -try: - from unittest import mock -except ImportError: - import mock - class XLS2XFormTests(TestCase): - survey_package = { - "id_string": "test_2011_08_29b", - "name_of_main_section": "gps", - "sections": { - "gps": { - "children": [{"name": "location", "type": "gps"}], - "name": "gps", - "type": "survey", - } - }, - "title": "test", - } - survey = pyxform.create_survey(**survey_package) - def test_create_parser_without_args(self): """Should exit when no args provided.""" with self.assertRaises(SystemExit): From ed585c3e98f2989f6698111df2412dd8498fafd8 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 01:21:19 +1100 Subject: [PATCH 05/22] dev: fix linter warnings from flake8-comprehensions rules - unnecessary calls to list() or dict(), redundant comprehensions --- pyproject.toml | 2 +- pyxform/builder.py | 19 +++++++------------ pyxform/parsing/instance_expression.py | 2 +- pyxform/survey.py | 14 ++++++-------- pyxform/utils.py | 2 +- pyxform/xform2json.py | 2 +- pyxform/xls2json.py | 18 +++++++++++------- pyxform/xls2json_backends.py | 4 ++-- tests/builder_tests.py | 18 ++++++------------ tests/j2x_question_tests.py | 2 +- tests/test_range.py | 2 +- 11 files changed, 38 insertions(+), 47 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0a432d00d..db2f32591 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ extend-select = [ # Potentially useful rule sets to enable in future: select = [ "B", # flake8-bugbear -# "C4", # flake8-comprehensions + "C4", # flake8-comprehensions "E", # pycodestyle error # "ERA", # eradicate (commented out code) "F", # pyflakes diff --git a/pyxform/builder.py b/pyxform/builder.py index c67b8fc63..765c94ab2 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -336,18 +336,13 @@ def _name_and_label_substitutions(question_template, column_headers): # dictionary by language to do substitutions. info_by_lang = {} if isinstance(column_headers[const.LABEL], dict): - info_by_lang = dict( - [ - ( - lang, - { - const.NAME: column_headers[const.NAME], - const.LABEL: column_headers[const.LABEL][lang], - }, - ) - for lang in column_headers[const.LABEL].keys() - ] - ) + info_by_lang = { + lang: { + const.NAME: column_headers[const.NAME], + const.LABEL: column_headers[const.LABEL][lang], + } + for lang in column_headers[const.LABEL].keys() + } result = question_template.copy() for key in result.keys(): diff --git a/pyxform/parsing/instance_expression.py b/pyxform/parsing/instance_expression.py index 1118c0211..9b908e0f8 100644 --- a/pyxform/parsing/instance_expression.py +++ b/pyxform/parsing/instance_expression.py @@ -91,7 +91,7 @@ def find_boundaries(xml_text: str) -> List[Tuple[int, int]]: # Pair up the boundaries [1, 2, 3, 4] -> [(1, 2), (3, 4)]. bounds = iter(boundaries) - pos_bounds = [(x, y) for x, y in zip(bounds, bounds)] + pos_bounds = list(zip(bounds, bounds)) return pos_bounds diff --git a/pyxform/survey.py b/pyxform/survey.py index 2fef5e788..e84eafd63 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -261,13 +261,11 @@ def get_nsmap(self): xmlns = "xmlns:" nsmap = NSMAP.copy() nsmap.update( - dict( - [ - (xmlns + k, v.replace('"', "").replace("'", "")) - for k, v in nslist - if xmlns + k not in nsmap - ] - ) + { + xmlns + k: v.replace('"', "").replace("'", "") + for k, v in nslist + if xmlns + k not in nsmap + } ) return nsmap @@ -619,7 +617,7 @@ def xml_model(self): model_children += self.xml_actions() if self.submission_url or self.public_key or self.auto_send or self.auto_delete: - submission_attrs = dict() + submission_attrs = {} if self.submission_url: submission_attrs["action"] = self.submission_url submission_attrs["method"] = "post" diff --git a/pyxform/utils.py b/pyxform/utils.py index 628bb6652..8d3ef53be 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -347,7 +347,7 @@ def levenshtein_distance(a: str, b: str) -> int: # initialize v0 (the previous row of distances) # this row is A[0][i]: edit distance for an empty s # the distance is just the number of characters to delete from t - v0 = [i for i in range(0, n + 1)] + v0 = list(range(0, n + 1)) for i in range(0, m): # calculate v1 (current row distances) from the previous row v0 diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 02e8ad319..2cba7fef1 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -68,7 +68,7 @@ def wrap(x): @staticmethod def _un_wrap(x): if isinstance(x, dict): - return dict((k, XmlDictObject._un_wrap(v)) for (k, v) in iter(x.items())) + return {k: XmlDictObject._un_wrap(v) for k, v in x.items()} elif isinstance(x, list): return [XmlDictObject._un_wrap(v) for v in x] else: diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index a07e8bc31..ed0eee329 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -72,7 +72,7 @@ def merge_dicts(dict_a, dict_b, default_key="default"): all_keys = {k: None for k in dict_a.keys()} all_keys.update({k: None for k in dict_b.keys()}) - out_dict = dict() + out_dict = {} for key in all_keys.keys(): out_dict[key] = merge_dicts(dict_a.get(key), dict_b.get(key), default_key) return out_dict @@ -131,10 +131,10 @@ def dealias_and_group_headers( without a language specified with localized versions. """ group_delimiter = "::" - out_dict_array = list() + out_dict_array = [] seen_headers = {} for row in dict_array: - out_row = dict() + out_row = {} for header, val in row.items(): if ignore_case: header = header.lower() @@ -230,7 +230,7 @@ def group_dictionaries_by_key(list_of_dicts, key, remove_key=True): The grouping key is removed by default. If the key is not in any dictionary an empty dict is returned. """ - dict_of_lists = dict() + dict_of_lists = {} for dicty in list_of_dicts: if key not in dicty: continue @@ -315,11 +315,15 @@ def process_range_question_type( if key not in parameters: parameters[key] = defaults[key] + has_float = False try: - has_float = any([float(x) and "." in str(x) for x in parameters.values()]) + # Check all parameters. + for x in parameters.values(): + if float(x) and "." in str(x): + has_float = True except ValueError as range_err: raise PyXFormError( - "Range parameters 'start', " "'end' or 'step' must all be numbers." + "Range parameters 'start', 'end' or 'step' must all be numbers." ) from range_err else: # is integer by default, convert to decimal if it has any float values @@ -1025,7 +1029,7 @@ def workbook_to_json( new_json_dict = row.copy() new_json_dict[constants.TYPE] = control_type - child_list = list() + child_list = [] new_json_dict[constants.CHILDREN] = child_list if control_type is constants.LOOP: if not parse_dict.get("list_name"): diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index e3960d832..fb341c4f8 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -53,7 +53,7 @@ def trim_trailing_empty(a_list: list, n_empty: int) -> list: def get_excel_column_headers(first_row: Iterator[Optional[str]]) -> List[Optional[str]]: """Get column headers from the first row; stop if there's a run of empty columns.""" max_adjacent_empty_columns = 20 - column_header_list = list() + column_header_list = [] adjacent_empty_cols = 0 for column_header in first_row: if is_empty(column_header): @@ -313,7 +313,7 @@ def replace_prefix(d, prefix): elif isinstance(v, dict): d[k] = replace_prefix(v, prefix) elif isinstance(v, list): - d[k] = map(lambda x: replace_prefix(x, prefix), v) + d[k] = (replace_prefix(x, prefix) for x in v) return d return_list.append(replace_prefix(row["lambda"], prefix)) diff --git a/tests/builder_tests.py b/tests/builder_tests.py index a87bd63bb..4ede8d8b3 100644 --- a/tests/builder_tests.py +++ b/tests/builder_tests.py @@ -550,12 +550,9 @@ def test_style_not_added_to_body_if_not_present(self): xml = survey.to_xml() # find the body tag root_elm = ETree.fromstring(xml.encode("utf-8")) - body_elms = list( - filter( - lambda e: self.STRIP_NS_FROM_TAG_RE.sub("", e.tag) == "body", - [c for c in root_elm], - ) - ) + body_elms = [ + c for c in root_elm if self.STRIP_NS_FROM_TAG_RE.sub("", c.tag) == "body" + ] self.assertEqual(len(body_elms), 1) self.assertIsNone(body_elms[0].get("class")) @@ -566,11 +563,8 @@ def test_style_added_to_body_if_present(self): xml = survey.to_xml() # find the body tag root_elm = ETree.fromstring(xml.encode("utf-8")) - body_elms = list( - filter( - lambda e: self.STRIP_NS_FROM_TAG_RE.sub("", e.tag) == "body", - [c for c in root_elm], - ) - ) + body_elms = [ + c for c in root_elm if self.STRIP_NS_FROM_TAG_RE.sub("", c.tag) == "body" + ] self.assertEqual(len(body_elms), 1) self.assertEqual(body_elms[0].get("class"), "ltr") diff --git a/tests/j2x_question_tests.py b/tests/j2x_question_tests.py index 35c3e24e7..2bcbcd240 100644 --- a/tests/j2x_question_tests.py +++ b/tests/j2x_question_tests.py @@ -178,7 +178,7 @@ def test_simple_phone_number_question_type_multilingual(self): "type": "string", "constraint": r"regex(., '^\d*$')", } - observed = {k: v for k, v in q.xml_bindings()[0].attributes.items()} + observed = {k: v for k, v in q.xml_bindings()[0].attributes.items()} # noqa: C416 self.assertDictEqual(expected, observed) def test_simple_select_all_question_multilingual(self): diff --git a/tests/test_range.py b/tests/test_range.py index 39402d58a..dde26bb10 100644 --- a/tests/test_range.py +++ b/tests/test_range.py @@ -89,7 +89,7 @@ def test_range_type_float(self): ], ) - def test_range_type_invvalid_parameters(self): + def test_range_type_invalid_parameters(self): # 'increment' is an invalid property self.assertPyxformXform( name="data", From bfa91e373c4dda6e38533c4c27362524833bbe81 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 01:37:49 +1100 Subject: [PATCH 06/22] dev: fix linter warnings from perflint and flake8-pie rules - unnecessary placeholders, unnecessary range starts (0), dict.values() - re-order ruff calls since `check` doesn't auto-format --- .github/workflows/verify.yml | 2 +- pre-commit.sh | 2 +- pyproject.toml | 6 ++++-- pyxform/errors.py | 4 ---- pyxform/external_instance.py | 1 - pyxform/utils.py | 8 ++++---- pyxform/validators/enketo_validate/__init__.py | 2 -- pyxform/validators/error_cleaner.py | 9 +++------ pyxform/validators/odk_validate/__init__.py | 2 -- pyxform/xls2json_backends.py | 2 +- tests/dump_and_load_tests.py | 4 ++-- tests/pyxform_test_case.py | 2 +- tests/xform2json_test.py | 4 ++-- 13 files changed, 19 insertions(+), 29 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4033be7ba..d4c3e44cb 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -30,8 +30,8 @@ jobs: pip list # Linter. - - run: ruff format pyxform tests --diff - run: ruff check pyxform tests --no-fix + - run: ruff format pyxform tests --diff test: runs-on: ${{ matrix.os }} diff --git a/pre-commit.sh b/pre-commit.sh index baf73600d..d959c5b4d 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -4,6 +4,6 @@ FILES=$(git diff --cached --name-only --diff-filter=ACM | grep -e '\.py$') if [ -n "$FILES" ]; then - ruff format pyxform tests ruff check pyxform tests + ruff format pyxform tests fi diff --git a/pyproject.toml b/pyproject.toml index db2f32591..5523d2bd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,8 +62,8 @@ select = [ # "ERA", # eradicate (commented out code) "F", # pyflakes "I", # isort -# "PERF", # perflint -# "PIE", # flake8-pie + "PERF", # perflint + "PIE", # flake8-pie # "PL", # pylint # "PTH", # flake8-use-pathlib # "PYI", # flake8-pyi @@ -78,6 +78,8 @@ select = [ ignore = [ "E501", # line-too-long (we have a lot of long strings) "F821", # undefined-name (doesn't work well with type hints as of ruff 0.1.11). + "PERF401", # manual-list-comprehension (false positives on selective transforms) + "PERF402", # manual-list-copy (false positives on selective transforms) "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) ] # per-file-ignores = {"tests/*" = ["E501"]} diff --git a/pyxform/errors.py b/pyxform/errors.py index 606b50707..3de3e37e9 100644 --- a/pyxform/errors.py +++ b/pyxform/errors.py @@ -7,10 +7,6 @@ class PyXFormError(Exception): """Common base class for pyxform exceptions.""" - pass - class ValidationError(PyXFormError): """Common base class for pyxform validation exceptions.""" - - pass diff --git a/pyxform/external_instance.py b/pyxform/external_instance.py index 1f0838037..c2a5d322b 100644 --- a/pyxform/external_instance.py +++ b/pyxform/external_instance.py @@ -12,4 +12,3 @@ def xml_control(self): Exists here because there's a soft abstractmethod in SurveyElement. """ - pass diff --git a/pyxform/utils.py b/pyxform/utils.py index 8d3ef53be..05ecc6f3d 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -342,14 +342,14 @@ def levenshtein_distance(a: str, b: str) -> int: n = len(b) # create two work vectors of integer distances - v1 = [0 for _ in range(0, n + 1)] + v1 = [0 for _ in range(n + 1)] # initialize v0 (the previous row of distances) # this row is A[0][i]: edit distance for an empty s # the distance is just the number of characters to delete from t - v0 = list(range(0, n + 1)) + v0 = list(range(n + 1)) - for i in range(0, m): + for i in range(m): # calculate v1 (current row distances) from the previous row v0 # first element of v1 is A[i+1][0] @@ -357,7 +357,7 @@ def levenshtein_distance(a: str, b: str) -> int: v1[0] = i + 1 # use formula to fill in the rest of the row - for j in range(0, n): + for j in range(n): # calculating costs for A[i+1][j+1] deletion_cost = v0[j + 1] + 1 insertion_cost = v1[j] + 1 diff --git a/pyxform/validators/enketo_validate/__init__.py b/pyxform/validators/enketo_validate/__init__.py index 931052bc1..e880ecac3 100644 --- a/pyxform/validators/enketo_validate/__init__.py +++ b/pyxform/validators/enketo_validate/__init__.py @@ -24,8 +24,6 @@ class EnketoValidateError(Exception): """Common base class for Enketo validate exceptions.""" - pass - def install_exists(): """ diff --git a/pyxform/validators/error_cleaner.py b/pyxform/validators/error_cleaner.py index 4b0080dcd..77a0ae1ba 100644 --- a/pyxform/validators/error_cleaner.py +++ b/pyxform/validators/error_cleaner.py @@ -13,12 +13,9 @@ def _replace_xpath_with_tokens(match): strmatch = match.group() # eliminate e.g /html/body/select1[@ref=/id_string/elId]/item/value # instance('q4')/root/item[...] - if ( - strmatch.startswith("/html/body") - or strmatch.startswith("/root/item") - or strmatch.startswith("/html/head/model/bind") - or strmatch.endswith("/item/value") - ): + if strmatch.startswith( + ("/html/body"), ("/root/item"), ("/html/head/model/bind") + ) or strmatch.endswith("/item/value"): return strmatch line = match.group().split("/") return "${%s}" % line[len(line) - 1] diff --git a/pyxform/validators/odk_validate/__init__.py b/pyxform/validators/odk_validate/__init__.py index 25dd47968..863c9d876 100644 --- a/pyxform/validators/odk_validate/__init__.py +++ b/pyxform/validators/odk_validate/__init__.py @@ -27,8 +27,6 @@ class ODKValidateError(Exception): """ODK Validation exception error.""" - pass - def install_exists(): """Returns True if ODK_VALIDATE_PATH exists.""" diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index fb341c4f8..69039484e 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -148,7 +148,7 @@ def xls_to_dict_normal_sheet(sheet): first_row = (c.value for c in next(sheet.get_rows(), [])) headers = get_excel_column_headers(first_row=first_row) row_iter = ( - tuple(sheet.cell(r, c) for c in range(0, len(headers))) + tuple(sheet.cell(r, c) for c in range(len(headers))) for r in range(1, sheet.nrows) ) rows = get_excel_rows(headers=headers, rows=row_iter, cell_func=xls_clean_cell) diff --git a/tests/dump_and_load_tests.py b/tests/dump_and_load_tests.py index 20def15e6..6c0169b25 100644 --- a/tests/dump_and_load_tests.py +++ b/tests/dump_and_load_tests.py @@ -33,13 +33,13 @@ def setUp(self): self.surveys[filename] = create_survey_from_path(path) def test_load_from_dump(self): - for _, survey in self.surveys.items(): + for survey in self.surveys.values(): survey.json_dump() path = survey.name + ".json" survey_from_dump = create_survey_from_path(path) self.assertEqual(survey.to_json_dict(), survey_from_dump.to_json_dict()) def tearDown(self): - for _, survey in self.surveys.items(): + for survey in self.surveys.values(): path = survey.name + ".json" os.remove(path) diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index a4c7880a8..56ee08cc8 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -85,7 +85,7 @@ def list_to_dicts(arr): def _row_to_dict(row): out_dict = {} - for i in range(0, len(row)): + for i in range(len(row)): col = row[i] if col not in [None, ""]: out_dict[headers[i]] = col diff --git a/tests/xform2json_test.py b/tests/xform2json_test.py index 81b1a105c..ba2ad8e0b 100644 --- a/tests/xform2json_test.py +++ b/tests/xform2json_test.py @@ -43,7 +43,7 @@ def setUp(self): self.surveys[filename] = create_survey_from_path(path) def test_load_from_dump(self): - for filename, survey in iter(self.surveys.items()): + for filename, survey in self.surveys.items(): with self.subTest(msg=filename): survey.json_dump() survey_from_dump = create_survey_element_from_xml(survey.to_xml()) @@ -52,7 +52,7 @@ def test_load_from_dump(self): self.assertXFormEqual(expected, observed) def tearDown(self): - for _, survey in self.surveys.items(): + for survey in self.surveys.values(): path = survey.name + ".json" if os.path.exists(path): os.remove(path) From 23a225f44a6108c73af66fa1ed349b29b0846a76 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 02:17:12 +1100 Subject: [PATCH 07/22] dev: fix linter warnings from pylint rules - collapsible else-ifs, collapsible multiple comparisons --- pyproject.toml | 18 ++++++++---- pyxform/question.py | 11 ++++---- pyxform/survey.py | 28 +++++++++---------- .../validators/enketo_validate/__init__.py | 21 +++++++------- pyxform/validators/error_cleaner.py | 2 +- pyxform/validators/odk_validate/__init__.py | 19 ++++++------- .../validators/pyxform/select_from_file.py | 1 - pyxform/validators/updater.py | 2 +- pyxform/validators/util.py | 7 +++-- pyxform/xls2json.py | 11 ++++---- tests/test_dynamic_default.py | 7 ++--- tests/utils.py | 9 +++--- 12 files changed, 67 insertions(+), 69 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5523d2bd0..97fd5aa93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ select = [ "I", # isort "PERF", # perflint "PIE", # flake8-pie -# "PL", # pylint + "PL", # pylint # "PTH", # flake8-use-pathlib # "PYI", # flake8-pyi # "RET", # flake8-return @@ -76,10 +76,16 @@ select = [ # "W", # pycodestyle warning ] ignore = [ - "E501", # line-too-long (we have a lot of long strings) - "F821", # undefined-name (doesn't work well with type hints as of ruff 0.1.11). - "PERF401", # manual-list-comprehension (false positives on selective transforms) - "PERF402", # manual-list-copy (false positives on selective transforms) - "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) + "E501", # line-too-long (we have a lot of long strings) + "F821", # undefined-name (doesn't work well with type hints as of ruff 0.1.11). + "PERF401", # manual-list-comprehension (false positives on selective transforms) + "PERF402", # manual-list-copy (false positives on selective transforms) + "PLR2004", # magic-value-comparison (many tests expect certain numbers of things) + "PLR0911", # too-many-return-statements (complexity not useful to warn every time) + "PLR0912", # too-many-branches (complexity not useful to warn every time) + "PLR0913", # too-many-arguments (complexity not useful to warn every time) + "PLR0915", # too-many-statements (complexity not useful to warn every time) + "PLW2901", # redefined-loop-name (usually not a bug) + "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) ] # per-file-ignores = {"tests/*" = ["E501"]} diff --git a/pyxform/question.py b/pyxform/question.py index 9fa5600e3..5144c8f93 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -239,13 +239,12 @@ def build_xml(self): ) if file_extension in EXTERNAL_INSTANCE_EXTENSIONS: - itemset = itemset + pass + elif not multi_language and not has_media and not has_dyn_label: + itemset = self["itemset"] else: - if not multi_language and not has_media and not has_dyn_label: - itemset = self["itemset"] - else: - itemset = self["itemset"] - itemset_label_ref = "jr:itext(itextId)" + itemset = self["itemset"] + itemset_label_ref = "jr:itext(itextId)" choice_filter = survey.insert_xpaths( choice_filter, self, True, is_previous_question diff --git a/pyxform/survey.py b/pyxform/survey.py index e84eafd63..f20775107 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -454,7 +454,7 @@ def _generate_from_file_instances(element) -> Optional[InstanceInfo]: file_id, ext = os.path.splitext(itemset) if itemset and ext in EXTERNAL_INSTANCE_EXTENSIONS: uri = "jr://%s/%s" % ( - "file" if ext == ".xml" or ext == ".geojson" else "file-%s" % ext[1:], + "file" if ext in {".xml", ".geojson"} else "file-%s" % ext[1:], itemset, ) return InstanceInfo( @@ -710,11 +710,10 @@ def _setup_choice_translations( if isinstance(value, dict): for language, val in value.items(): yield ([language, itext_id, media_or_lang], val) + elif name == constants.MEDIA: + yield ([self.default_language, itext_id, media_or_lang], value) else: - if name == constants.MEDIA: - yield ([self.default_language, itext_id, media_or_lang], value) - else: - yield ([media_or_lang, itext_id, "long"], value) + yield ([media_or_lang, itext_id, "long"], value) itemsets_multi_language = set() itemsets_has_media = set() @@ -926,7 +925,7 @@ def itext(self): itext_nodes.append( node("value", value, toParseString=output_inserted) ) - elif media_type == "image" or media_type == "big-image": + elif media_type in {"image", "big-image"}: if value != "-": itext_nodes.append( node( @@ -936,16 +935,15 @@ def itext(self): toParseString=output_inserted, ) ) - else: - if value != "-": - itext_nodes.append( - node( - "value", - "jr://" + media_type + "/" + value, - form=media_type, - toParseString=output_inserted, - ) + elif value != "-": + itext_nodes.append( + node( + "value", + "jr://" + media_type + "/" + value, + form=media_type, + toParseString=output_inserted, ) + ) result[-1].appendChild(node("text", *itext_nodes, id=label_name)) diff --git a/pyxform/validators/enketo_validate/__init__.py b/pyxform/validators/enketo_validate/__init__.py index e880ecac3..965005b77 100644 --- a/pyxform/validators/enketo_validate/__init__.py +++ b/pyxform/validators/enketo_validate/__init__.py @@ -73,14 +73,13 @@ def check_xform(path_to_xform): if timeout: return ["XForm took to long to completely validate."] - else: - if returncode > 0: # Error invalid - raise EnketoValidateError( - "Enketo Validate Errors:\n" + ErrorCleaner.enketo_validate(stderr) - ) - elif returncode == 0: - if stdout: - warnings.append("Enketo Validate Warnings:\n" + stdout) - return warnings - elif returncode < 0: - return ["Bad return code from Enketo Validate."] + elif returncode > 0: # Error invalid + raise EnketoValidateError( + "Enketo Validate Errors:\n" + ErrorCleaner.enketo_validate(stderr) + ) + elif returncode == 0: + if stdout: + warnings.append("Enketo Validate Warnings:\n" + stdout) + return warnings + elif returncode < 0: + return ["Bad return code from Enketo Validate."] diff --git a/pyxform/validators/error_cleaner.py b/pyxform/validators/error_cleaner.py index 77a0ae1ba..74a7d738d 100644 --- a/pyxform/validators/error_cleaner.py +++ b/pyxform/validators/error_cleaner.py @@ -14,7 +14,7 @@ def _replace_xpath_with_tokens(match): # eliminate e.g /html/body/select1[@ref=/id_string/elId]/item/value # instance('q4')/root/item[...] if strmatch.startswith( - ("/html/body"), ("/root/item"), ("/html/head/model/bind") + (("/html/body"), ("/root/item"), ("/html/head/model/bind")) ) or strmatch.endswith("/item/value"): return strmatch line = match.group().split("/") diff --git a/pyxform/validators/odk_validate/__init__.py b/pyxform/validators/odk_validate/__init__.py index 863c9d876..fe9791498 100644 --- a/pyxform/validators/odk_validate/__init__.py +++ b/pyxform/validators/odk_validate/__init__.py @@ -90,16 +90,15 @@ def check_xform(path_to_xform): if result.timeout: return ["XForm took to long to completely validate."] - else: - if result.return_code > 0: # Error invalid - raise ODKValidateError( - "ODK Validate Errors:\n" + ErrorCleaner.odk_validate(result.stderr) - ) - elif result.return_code == 0: - if result.stderr: - warnings.append("ODK Validate Warnings:\n" + result.stderr) - elif result.return_code < 0: - return ["Bad return code from ODK Validate."] + elif result.return_code > 0: # Error invalid + raise ODKValidateError( + "ODK Validate Errors:\n" + ErrorCleaner.odk_validate(result.stderr) + ) + elif result.return_code == 0: + if result.stderr: + warnings.append("ODK Validate Warnings:\n" + result.stderr) + elif result.return_code < 0: + return ["Bad return code from ODK Validate."] return warnings diff --git a/pyxform/validators/pyxform/select_from_file.py b/pyxform/validators/pyxform/select_from_file.py index 161b76b59..be29be70c 100644 --- a/pyxform/validators/pyxform/select_from_file.py +++ b/pyxform/validators/pyxform/select_from_file.py @@ -43,7 +43,6 @@ def value_or_label_check(name: str, value: str, row_number: int) -> None: if not value_or_label_test(value=value): msg = value_or_label_format_msg(name=name, row_number=row_number) raise PyXFormError(msg) - return None def validate_list_name_extension( diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index c5303b962..fa503ea04 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -393,7 +393,7 @@ def _install(update_info, file_name): os.chmod(new_bin_file_path, current_mode | S_IXUSR | S_IXGRP) except PyXFormError as px_err: - raise PyXFormError("\n\nUpdate failed!\n\n" + str(e)) from px_err + raise PyXFormError("\n\nUpdate failed!\n\n" + str(px_err)) from px_err else: return latest diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index e9dee5a63..2036cd088 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -48,7 +48,6 @@ def _kill_process_after_a_timeout(pid): os.kill(pid, signal.SIGTERM) kill_check.set() # tell the main routine that we had to kill # use SIGKILL if hard to kill... - return startup_info = None env = None @@ -121,11 +120,13 @@ def request_get(url): raise PyXFormError( "Unable to fulfill request. Error code: '{c}'. " "Reason: '{r}'. URL: '{u}'." - "".format(r=e.reason, c=e.code, u=url) + "".format(r=http_err.reason, c=http_err.code, u=url) ) from http_err except URLError as url_err: raise PyXFormError( - "Unable to reach a server. Reason: {r}. " "URL: {u}".format(r=e.reason, u=url) + "Unable to reach a server. Reason: {r}. " "URL: {u}".format( + r=url_err.reason, u=url + ) ) from url_err diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index ed0eee329..04b69c426 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -287,12 +287,11 @@ def add_flat_annotations(prompt_list, parent_relevant="", name_prefix=""): add_flat_annotations( children, new_relevant, name_prefix + "_" + prompt["name"] ) - else: - if new_relevant != "": - prompt["bind"] = prompt.get("bind", {}) - prompt["bind"]["relevant"] = new_relevant - # if name_prefix != '': - # prompt['name'] = name_prefix + prompt['name'] + elif new_relevant != "": + prompt["bind"] = prompt.get("bind", {}) + prompt["bind"]["relevant"] = new_relevant + # if name_prefix != '': + # prompt['name'] = name_prefix + prompt['name'] def process_range_question_type( diff --git a/tests/test_dynamic_default.py b/tests/test_dynamic_default.py index 95309ef74..ad8c6f0b1 100644 --- a/tests/test_dynamic_default.py +++ b/tests/test_dynamic_default.py @@ -97,11 +97,10 @@ def model(q_num: int, case: Case): else: if 0 == len(q_default_final): q_default_cmp = """and not(text()) """ + elif "'" in q_default_final: + q_default_cmp = "" else: - if "'" in q_default_final: - q_default_cmp = "" - else: - q_default_cmp = f"""and text()='{q_default_final}' """ + q_default_cmp = f"""and text()='{q_default_final}' """ return rf""" /h:html/h:head/x:model /x:instance/x:test_name[@id="test_id"]/x:q{q_num}[ diff --git a/tests/utils.py b/tests/utils.py index d1cece88c..e2828c3cf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -94,11 +94,10 @@ def truncate_temp_files(temp_dir): for f in os.scandir(temp_dir): if os.path.isdir(f.path): truncate_temp_files(f.path) - else: - # Check still in temp directory - if f.path.startswith(temp_root): - with open(f.path, mode="w") as _: - pass + # Check still in temp directory + elif f.path.startswith(temp_root): + with open(f.path, mode="w") as _: + pass def cleanup_pyxform_temp_files(prefix: str): From 5c680e770a55c6dce73e577d2cea633e9fa011d1 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 03:04:01 +1100 Subject: [PATCH 08/22] dev: fix linter warnings from flake8-bandit rules - use of assert, use of ElementTree, unchecked URL scheme --- pyproject.toml | 7 +++-- pyxform/builder.py | 3 +- pyxform/question.py | 3 +- pyxform/utils.py | 5 ++- pyxform/validators/util.py | 2 ++ pyxform/xform2json.py | 52 ++++++++++++++++++++------------ pyxform/xform_instance_parser.py | 23 +++++++++----- pyxform/xls2json.py | 22 -------------- tests/builder_tests.py | 2 +- tests/pyxform_test_case.py | 3 +- tests/xform_test_case/base.py | 6 ++-- 11 files changed, 69 insertions(+), 59 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97fd5aa93..ee774a4d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,7 @@ select = [ # "PYI", # flake8-pyi # "RET", # flake8-return "RUF", # ruff-specific rules -# "S", # flake8-bandit + "S", # flake8-bandit # "SIM", # flake8-simplify # "TRY", # tryceratops # "UP", # pyupgrade @@ -77,7 +77,7 @@ select = [ ] ignore = [ "E501", # line-too-long (we have a lot of long strings) - "F821", # undefined-name (doesn't work well with type hints as of ruff 0.1.11). + "F821", # undefined-name (doesn't work well with type hints, ruff 0.1.11). "PERF401", # manual-list-comprehension (false positives on selective transforms) "PERF402", # manual-list-copy (false positives on selective transforms) "PLR2004", # magic-value-comparison (many tests expect certain numbers of things) @@ -86,6 +86,9 @@ ignore = [ "PLR0913", # too-many-arguments (complexity not useful to warn every time) "PLR0915", # too-many-statements (complexity not useful to warn every time) "PLW2901", # redefined-loop-name (usually not a bug) + "S310", # suspicious-url-open-usage (prone to false positives, ruff 0.1.11) + "S320", # suspicious-xmle-tree-usage (according to defusedxml author lxml is safe) + "S603", # subprocess-without-shell-equals-true (prone to false positives, ruff 0.1.11) "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) ] # per-file-ignores = {"tests/*" = ["E501"]} diff --git a/pyxform/builder.py b/pyxform/builder.py index 765c94ab2..f1fabeb5d 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -96,7 +96,8 @@ def set_sections(self, sections): the name of the section and the value is a dict that can be used to create a whole survey. """ - assert isinstance(sections, dict) + if not isinstance(sections, dict): + raise PyXFormError("""Invalid value for `sections`.""") self._sections = sections def create_survey_element_from_dict( diff --git a/pyxform/question.py b/pyxform/question.py index 5144c8f93..4ddca1cb2 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -200,7 +200,8 @@ def validate(self): choice.validate() def build_xml(self): - assert self.bind["type"] in ["string", "odk:rank"] + if self.bind["type"] not in ["string", "odk:rank"]: + raise PyXFormError("""Invalid value for `self.bind["type"]`.""") survey = self.get_root() control_dict = self.control.copy() # Resolve field references in attributes diff --git a/pyxform/utils.py b/pyxform/utils.py index 05ecc6f3d..4f3229474 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -16,7 +16,9 @@ import openpyxl import xlrd +from defusedxml.minidom import parseString +from pyxform.errors import PyXFormError from pyxform.xls2json_backends import is_empty, xls_value_to_unicode, xlsx_value_to_str SEP = "_" @@ -112,7 +114,8 @@ def node(*args, **kwargs) -> DetachableElement: args = args[1:] result = DetachableElement(tag) unicode_args = [u for u in args if isinstance(u, str)] - assert len(unicode_args) <= 1 + if len(unicode_args) > 1: + raise PyXFormError("""Invalid value for `unicode_args`.""") parsed_string = False # Convert the kwargs xml attribute dictionary to a xml.dom.minidom.Element. diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index 2036cd088..c0ca5c88a 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -108,6 +108,8 @@ def request_get(url): Get the response content from URL. """ try: + if not url.startswith(("http:", "https:")): + raise ValueError("URL must start with 'http:' or 'https:'") r = Request(url) r.add_header("Accept", "application/json") with closing(urlopen(r)) as u: diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 2cba7fef1..118ee7202 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -7,10 +7,12 @@ import json import logging import re -import xml.etree.ElementTree as ETree from collections import Mapping from operator import itemgetter from typing import Any, Dict, List +from xml.etree.ElementTree import Element + +from defusedxml.ElementTree import ParseError, XMLParser, fromstring, parse from pyxform import builder from pyxform.errors import PyXFormError @@ -84,7 +86,8 @@ def un_wrap(self): def _convert_dict_to_xml_recurse(parent, dictitem): - assert not isinstance(dictitem, list) + if isinstance(dictitem, list): + raise PyXFormError("""Invalid value for `dictitem`.""") if isinstance(dictitem, dict): for tag, child in iter(dictitem.items()): @@ -93,11 +96,11 @@ def _convert_dict_to_xml_recurse(parent, dictitem): elif isinstance(child, list): # iterate through the array and convert for listchild in child: - elem = ETree.Element(tag) + elem = Element(tag) parent.append(elem) _convert_dict_to_xml_recurse(elem, listchild) else: - elem = ETree.Element(tag) + elem = Element(tag) parent.append(elem) _convert_dict_to_xml_recurse(elem, child) else: @@ -110,7 +113,7 @@ def convert_dict_to_xml(xmldict): """ roottag = xmldict.keys()[0] - root = ETree.Element(roottag) + root = Element(roottag) _convert_dict_to_xml_recurse(root, xmldict[roottag]) return root @@ -164,7 +167,7 @@ def convert_xml_to_dict(root, dictclass=XmlDictObject): # If a string is passed in, try to open it as a file if isinstance(root, str): root = _try_parse(root) - elif not isinstance(root, ETree.Element): + elif not isinstance(root, Element): raise TypeError("Expected ElementTree.Element or file path string") return dictclass({root.tag: _convert_xml_to_dict_recurse(root, dictclass)}) @@ -179,21 +182,21 @@ def _try_parse(root, parser=None): """ root = root.encode("UTF-8") try: - parsed_root = ETree.fromstring(root, parser) - except ETree.ParseError: - parsed_root = ETree.parse(root, parser=parser).getroot() + parsed_root = fromstring(root, parser) + except ParseError: + parsed_root = parse(root, parser=parser).getroot() return parsed_root class XFormToDict: def __init__(self, root): if isinstance(root, str): - parser = ETree.XMLParser(encoding="UTF-8") + parser = XMLParser(encoding="UTF-8") self._root = _try_parse(root, parser) self._dict = XmlDictObject( {self._root.tag: _convert_xml_to_dict_recurse(self._root, XmlDictObject)} ) - elif not isinstance(root, ETree.Element): + elif not isinstance(root, Element): raise TypeError("Expected ElementTree.Element or file path string") def get_dict(self): @@ -215,12 +218,18 @@ def __init__(self, xml_file): doc_as_dict = XFormToDict(xml_file).get_dict() self._xmldict = doc_as_dict - assert "html" in doc_as_dict - assert "body" in doc_as_dict["html"] - assert "head" in doc_as_dict["html"] - assert "model" in doc_as_dict["html"]["head"] - assert "title" in doc_as_dict["html"]["head"] - assert "bind" in doc_as_dict["html"]["head"]["model"] + if "html" not in doc_as_dict: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "body" not in doc_as_dict["html"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "head" not in doc_as_dict["html"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "model" not in doc_as_dict["html"]["head"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "title" not in doc_as_dict["html"]["head"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "bind" not in doc_as_dict["html"]["head"]["model"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") self.body = doc_as_dict["html"]["body"] self.model = doc_as_dict["html"]["head"]["model"] @@ -558,12 +567,15 @@ def _get_question_type(question_type): def _get_translations(self) -> List[Dict]: if "itext" not in self.model: return [] - assert "translation" in self.model["itext"] + if "translation" not in self.model["itext"]: + raise PyXFormError("""Invalid value for `self.model["itext"]`.""") translations = self.model["itext"]["translation"] if isinstance(translations, dict): translations = [translations] - assert "text" in translations[0] - assert "lang" in translations[0] + if "text" not in translations[0]: + raise PyXFormError("""Invalid value for `translations[0]`.""") + if "lang" not in translations[0]: + raise PyXFormError("""Invalid value for `translations[0]`.""") return translations def _get_label(self, label_obj, key="label"): diff --git a/pyxform/xform_instance_parser.py b/pyxform/xform_instance_parser.py index e16d257f3..6eca12dc5 100644 --- a/pyxform/xform_instance_parser.py +++ b/pyxform/xform_instance_parser.py @@ -6,13 +6,18 @@ # where this code is actually going to live. import re -from xml.dom import minidom +from xml.dom.minidom import Node + +from defusedxml.minidom import parseString + +from pyxform.errors import PyXFormError XFORM_ID_STRING = "_xform_id_string" def _xml_node_to_dict(node): - assert isinstance(node, minidom.Node) + if not isinstance(node, Node): + raise PyXFormError("""Invalid value for `node`.""") if len(node.childNodes) == 0: # there's no data for this leaf node value = None @@ -25,7 +30,8 @@ def _xml_node_to_dict(node): for child in node.childNodes: d = _xml_node_to_dict(child) child_name = child.nodeName - assert list(d.keys()) == [child_name] + if list(d.keys()) != [child_name]: + raise PyXFormError("""Invalid value for `d`.""") if child_name not in value: # copy the value into the dict value[child_name] = d[child_name] @@ -42,8 +48,10 @@ def _flatten_dict(d, prefix): """ Return a list of XPath, value pairs. """ - assert isinstance(d, dict) - assert isinstance(prefix, list) + if not isinstance(d, dict): + raise PyXFormError("""Invalid value for `d`.""") + if not isinstance(prefix, list): + raise PyXFormError("""Invalid value for `prefix`.""") for key, value in d.items(): new_prefix = [*prefix, key] @@ -86,7 +94,7 @@ def __init__(self, xml_str): def parse(self, xml_str): clean_xml_str = xml_str.strip() clean_xml_str = re.sub(str(r">\s+<"), str("><"), clean_xml_str) - self._xml_obj = minidom.parseString(clean_xml_str) + self._xml_obj = parseString(clean_xml_str) self._root_node = self._xml_obj.documentElement self._dict = _xml_node_to_dict(self._root_node) self._flat_dict = {} @@ -113,7 +121,8 @@ def _set_attributes(self): self._attributes = {} all_attributes = list(_get_all_attributes(self._root_node)) for key, value in all_attributes: - assert key not in self._attributes + if key in self._attributes: + raise PyXFormError("""Invalid value for `self._attributes`.""") self._attributes[key] = value def get_xform_id_string(self): diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 04b69c426..862952202 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -1715,28 +1715,6 @@ def _setup_question_types_dictionary(self): self._dict = organize_by_values(self._dict, "name") -# Not used internally (or on formhub) -# TODO: If this is used anywhere else it is probably broken -# from the changes I made to SpreadsheetReader. -class VariableNameReader(SpreadsheetReader): - def __init__(self, path): - SpreadsheetReader.__init__(self, path) - self._organize_renames() - - def _organize_renames(self): - new_dict = {} - variable_names_so_far = [] - assert "Dictionary" in self._dict - for d in self._dict["Dictionary"]: - if "Variable Name" in d: - assert d["Variable Name"] not in variable_names_so_far, d["Variable Name"] - variable_names_so_far.append(d["Variable Name"]) - new_dict[d["XPath"]] = d["Variable Name"] - else: - variable_names_so_far.append(d["XPath"]) - self._dict = new_dict - - if __name__ == "__main__": # Open the excel file specified by the argument of this python call, # convert that file to json, then print it diff --git a/tests/builder_tests.py b/tests/builder_tests.py index 4ede8d8b3..bbadeeba2 100644 --- a/tests/builder_tests.py +++ b/tests/builder_tests.py @@ -4,9 +4,9 @@ """ import os import re -import xml.etree.ElementTree as ETree from unittest import TestCase +import defusedxml.ElementTree as ETree from pyxform import InputQuestion, Survey from pyxform.builder import SurveyElementBuilder, create_survey_from_xls from pyxform.errors import PyXFormError diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 56ee08cc8..22aa273a4 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -139,7 +139,8 @@ def _run_odk_validate(xml): finally: # Clean up the temporary file os.remove(tmp.name) - assert not os.path.isfile(tmp.name) + if os.path.isfile(tmp.name): + raise PyXFormError(f"Temporary file still exists: {tmp.name}") @staticmethod def _autoname_inputs( diff --git a/tests/xform_test_case/base.py b/tests/xform_test_case/base.py index 791c023ae..b99dd0314 100644 --- a/tests/xform_test_case/base.py +++ b/tests/xform_test_case/base.py @@ -1,7 +1,7 @@ import os -import xml.etree.ElementTree as ETree from unittest import TestCase +from defusedxml.ElementTree import fromstring from formencode.doctest_xml_compare import xml_compare from tests import example_xls, test_output @@ -28,8 +28,8 @@ def get_file_path(self, filename): self.output_path = os.path.join(test_output.PATH, self.root_filename + ".xml") def assertXFormEqual(self, xform1, xform2): - xform1 = ETree.fromstring(xform1.encode("utf-8")) - xform2 = ETree.fromstring(xform2.encode("utf-8")) + xform1 = fromstring(xform1.encode("utf-8")) + xform2 = fromstring(xform2.encode("utf-8")) # Sort tags under section in each form self.sort_model(xform1) From abf15e2e5ec0320a70bce3e60523ba0b1c2640fa Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 03:31:38 +1100 Subject: [PATCH 09/22] dev: fix linter warnings from flake8-pyi, tryceratops rules - use type-able NamedTuple, raise accurate errors, log exception TBs --- pyproject.toml | 11 ++++++----- pyxform/instance.py | 3 ++- pyxform/survey.py | 6 +++--- pyxform/utils.py | 11 ++++++++--- pyxform/validators/util.py | 8 ++++++-- pyxform/xls2json.py | 2 +- pyxform/xls2xform.py | 8 ++++---- tests/pyxform_test_case.py | 4 ++-- tests/validators/server.py | 4 ++-- 9 files changed, 34 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee774a4d6..542fd9da9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,29 +66,30 @@ select = [ "PIE", # flake8-pie "PL", # pylint # "PTH", # flake8-use-pathlib -# "PYI", # flake8-pyi + "PYI", # flake8-pyi # "RET", # flake8-return "RUF", # ruff-specific rules "S", # flake8-bandit # "SIM", # flake8-simplify -# "TRY", # tryceratops + "TRY", # tryceratops # "UP", # pyupgrade -# "W", # pycodestyle warning + "W", # pycodestyle warning ] ignore = [ "E501", # line-too-long (we have a lot of long strings) "F821", # undefined-name (doesn't work well with type hints, ruff 0.1.11). "PERF401", # manual-list-comprehension (false positives on selective transforms) "PERF402", # manual-list-copy (false positives on selective transforms) - "PLR2004", # magic-value-comparison (many tests expect certain numbers of things) "PLR0911", # too-many-return-statements (complexity not useful to warn every time) "PLR0912", # too-many-branches (complexity not useful to warn every time) "PLR0913", # too-many-arguments (complexity not useful to warn every time) "PLR0915", # too-many-statements (complexity not useful to warn every time) + "PLR2004", # magic-value-comparison (many tests expect certain numbers of things) "PLW2901", # redefined-loop-name (usually not a bug) + "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) "S310", # suspicious-url-open-usage (prone to false positives, ruff 0.1.11) "S320", # suspicious-xmle-tree-usage (according to defusedxml author lxml is safe) "S603", # subprocess-without-shell-equals-true (prone to false positives, ruff 0.1.11) - "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) + "TRY003", # raise-vanilla-args (reasonable lint but would require large refactor) ] # per-file-ignores = {"tests/*" = ["E501"]} diff --git a/pyxform/instance.py b/pyxform/instance.py index b4ab9a17c..8845b39a0 100644 --- a/pyxform/instance.py +++ b/pyxform/instance.py @@ -2,6 +2,7 @@ """ SurveyInstance class module. """ +from pyxform.errors import PyXFormError from pyxform.xform_instance_parser import parse_xform_instance @@ -35,7 +36,7 @@ def xpaths(self): def answer(self, name=None, value=None): if name is None: - raise Exception("In answering, name must be given") + raise PyXFormError("In answering, name must be given") # ahh. this is horrible, but we need the xpath dict in survey # to be up-to-date ...maybe diff --git a/pyxform/survey.py b/pyxform/survey.py index f20775107..af2d5e820 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -890,7 +890,7 @@ def itext(self): label_type = label_name.partition(":")[-1] if not isinstance(content, dict): - raise Exception() + raise PyXFormError("""Invalid value for `content`.""") for media_type, media_value in content.items(): # Ignore key indicating Question or Choice translation type. @@ -1172,10 +1172,10 @@ def print_xform_to_file( file_obj.write(self._to_pretty_xml()) else: file_obj.write(self._to_ugly_xml()) - except Exception as error: + except Exception: if os.path.exists(path): os.unlink(path) - raise error + raise if validate: warnings.extend(odk_validate.check_xform(path)) if enketo: diff --git a/pyxform/utils.py b/pyxform/utils.py index 4f3229474..7d18e1bd9 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -8,9 +8,8 @@ import json import os import re -from collections import namedtuple from json.decoder import JSONDecodeError -from typing import Dict, List, Tuple +from typing import Dict, List, NamedTuple, Tuple from xml.dom import Node from xml.dom.minidom import Element, Text, _write_data, parseString @@ -446,7 +445,13 @@ def tokenizer(scan, value): # Scanner takes a few 100ms to compile so use this shared instance. -ExpLexerToken = namedtuple("ExpLexerToken", ["name", "value", "start", "end"]) +class ExpLexerToken(NamedTuple): + name: str + value: str + start: int + end: int + + EXPRESSION_LEXER = get_expression_lexer() diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index c0ca5c88a..7e3b86dc4 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -2,7 +2,6 @@ """ The validators utility functions. """ -import collections import io import logging import os @@ -13,6 +12,7 @@ import time from contextlib import closing from subprocess import PIPE, Popen +from typing import Dict, List, NamedTuple from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -132,6 +132,11 @@ def request_get(url): ) from url_err +class _LoggingWatcher(NamedTuple): + records: List + output: Dict + + class CapturingHandler(logging.Handler): """ A logging handler capturing all (raw and formatted) logging output. @@ -169,7 +174,6 @@ def emit(self, record): @staticmethod def _get_watcher(): - _LoggingWatcher = collections.namedtuple("_LoggingWatcher", ["records", "output"]) levels = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"] return _LoggingWatcher([], {x: [] for x in levels}) diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 862952202..de0f2b0e7 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -1634,7 +1634,7 @@ def organize_by_values(dict_list, key): dicty_copy = dicty.copy() val = dicty_copy.pop(key) if val in result: - raise Exception("Duplicate key: " + val) + raise PyXFormError("Duplicate key: " + val) result[val] = dicty_copy return result diff --git a/pyxform/xls2xform.py b/pyxform/xls2xform.py index feb6e75d5..c47c80d38 100644 --- a/pyxform/xls2xform.py +++ b/pyxform/xls2xform.py @@ -174,13 +174,13 @@ def main_cli(): pretty_print=args.pretty_print, enketo=args.enketo_validate, ) - except EnvironmentError as e: + except EnvironmentError: # Do not crash if 'java' not installed - logger.error(e) - except ODKValidateError as e: + logger.exception("EnvironmentError during conversion") + except ODKValidateError: # Remove output file if there is an error os.remove(args.output_path) - logger.error(e) + logger.exception("ODKValidateError during conversion.") else: if len(warnings) > 0: logger.warning("Warnings:") diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 22aa273a4..4738233f8 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -492,7 +492,7 @@ def assert_xpath_exact( """ if not (isinstance(xpath, str) and isinstance(expected, set)): msg = "Each xpath_exact requires: tuple(xpath: str, expected: Set[str])." - raise SyntaxError(msg) + raise TypeError(msg) observed = xpath_evaluate( matcher_context=matcher_context, content=content, @@ -524,7 +524,7 @@ def assert_xpath_count( """ if not (isinstance(xpath, str) and isinstance(expected, int)): msg = "Each xpath_count requires: tuple(xpath: str, count: int)" - raise SyntaxError(msg) + raise TypeError(msg) observed = xpath_evaluate( matcher_context=matcher_context, content=content, diff --git a/tests/validators/server.py b/tests/validators/server.py index 8acb9ee78..0847817e5 100644 --- a/tests/validators/server.py +++ b/tests/validators/server.py @@ -76,10 +76,10 @@ def _bind_and_activate(self, tries: int = 0): try: self.httpd.server_bind() self.httpd.server_activate() - except OSError as e: + except OSError: self.httpd.server_close() if 5 < tries: - raise e + raise else: time.sleep(0.5) self._bind_and_activate(tries=tries) From c5902b519d165e57e921d184ff422635044b80f6 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 03:37:54 +1100 Subject: [PATCH 10/22] dev: fix linter warnings from pyupgrade rules - UP009 utf8-encoding-declaration unnecessary for python 3 --- pyproject.toml | 1 + pyxform/aliases.py | 1 - pyxform/builder.py | 1 - pyxform/constants.py | 1 - pyxform/entities/entity_declaration.py | 2 -- pyxform/errors.py | 1 - pyxform/external_instance.py | 1 - pyxform/file_utils.py | 1 - pyxform/instance.py | 1 - pyxform/question.py | 1 - pyxform/question_type_dictionary.py | 1 - pyxform/section.py | 1 - pyxform/survey.py | 1 - pyxform/survey_element.py | 1 - pyxform/translator.py | 1 - pyxform/utils.py | 1 - pyxform/validators/enketo_validate/__init__.py | 1 - pyxform/validators/error_cleaner.py | 1 - pyxform/validators/odk_validate/__init__.py | 1 - pyxform/validators/updater.py | 1 - pyxform/validators/util.py | 1 - pyxform/xform2json.py | 1 - pyxform/xform_instance_parser.py | 1 - pyxform/xls2json.py | 1 - pyxform/xls2json_backends.py | 1 - pyxform/xls2xform.py | 1 - tests/builder_tests.py | 1 - tests/dump_and_load_tests.py | 1 - tests/file_utils_test.py | 1 - tests/group_test.py | 1 - tests/j2x_question_tests.py | 1 - tests/j2x_test_creation.py | 1 - tests/j2x_test_instantiation.py | 1 - tests/j2x_test_xform_build_preparation.py | 1 - tests/js2x_test_import_from_json.py | 1 - tests/json2xform_test.py | 1 - tests/loop_tests.py | 1 - tests/pyxform_test_case.py | 1 - tests/settings_test.py | 1 - tests/test_allow_mock_accuracy.py | 1 - tests/test_area.py | 1 - tests/test_audio_quality.py | 1 - tests/test_audit.py | 1 - tests/test_background_audio.py | 1 - tests/test_bind_conversions.py | 1 - tests/test_bug_missing_headers.py | 1 - tests/test_bug_round_calculation.py | 1 - tests/test_choices_sheet.py | 1 - tests/test_custom_xml_namespaces.py | 1 - tests/test_dynamic_default.py | 1 - tests/test_entities_create.py | 1 - tests/test_entities_update.py | 1 - tests/test_external_instances.py | 1 - tests/test_external_instances_for_selects.py | 1 - tests/test_fieldlist_labels.py | 1 - tests/test_fields.py | 1 - tests/test_file.py | 1 - tests/test_for_loop.py | 1 - tests/test_form_name.py | 1 - tests/test_geo.py | 1 - tests/test_groups.py | 1 - tests/test_guidance_hint.py | 1 - tests/test_image_app_parameter.py | 1 - tests/test_language_warnings.py | 1 - tests/test_last_saved.py | 1 - tests/test_metadata.py | 1 - tests/test_osm.py | 1 - tests/test_parameters_rows.py | 1 - tests/test_pyxformtestcase.py | 1 - tests/test_randomize_itemsets.py | 1 - tests/test_range.py | 1 - tests/test_rank.py | 1 - tests/test_repeat.py | 1 - tests/test_repeat_template.py | 1 - tests/test_secondary_instance_translations.py | 1 - tests/test_set_geopoint.py | 1 - tests/test_settings_auto_send_delete.py | 1 - tests/test_sheet_columns.py | 1 - tests/test_sms.py | 1 - tests/test_table_list.py | 1 - tests/test_translations.py | 1 - tests/test_trigger.py | 1 - tests/test_typed_calculates.py | 1 - tests/test_unicode_rtl.py | 1 - tests/test_utils/md_table.py | 1 - tests/test_validate_unicode_exception.py | 1 - tests/test_validator_update.py | 1 - tests/test_validator_util.py | 1 - tests/test_validators.py | 1 - tests/test_warnings.py | 1 - tests/test_whitespace.py | 1 - tests/test_xform2json.py | 1 - tests/test_xls2json_backends.py | 1 - tests/test_xlsform_headers.py | 1 - tests/test_xml_structure.py | 1 - tests/tutorial_test.py | 1 - tests/utils.py | 1 - tests/validators/__init__.py | 1 - tests/validators/server.py | 1 - tests/xform2json_test.py | 1 - tests/xform_test_case/attributecolumnstest.py | 1 - tests/xform_test_case/bug_tests.py | 1 - tests/xform_test_case/xlsform_spec_test.py | 1 - tests/xform_test_case/xml_tests.py | 1 - tests/xls2json_tests.py | 1 - tests/xls2xform_tests.py | 1 - 106 files changed, 1 insertion(+), 106 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 542fd9da9..f9120c8a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ select = [ # "SIM", # flake8-simplify "TRY", # tryceratops # "UP", # pyupgrade + "UP009", "W", # pycodestyle warning ] ignore = [ diff --git a/pyxform/aliases.py b/pyxform/aliases.py index 710c68355..ede93c6fc 100644 --- a/pyxform/aliases.py +++ b/pyxform/aliases.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Aliases for elements which could mean the same element in XForm but is represented differently on the XLSForm. diff --git a/pyxform/builder.py b/pyxform/builder.py index f1fabeb5d..0f3dc2910 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Survey builder functionality. """ diff --git a/pyxform/constants.py b/pyxform/constants.py index 61c7684fe..f3821e44c 100644 --- a/pyxform/constants.py +++ b/pyxform/constants.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This file contains constants that correspond with the property names in the json survey format. (@see json_form_schema.json) These names are to be shared diff --git a/pyxform/entities/entity_declaration.py b/pyxform/entities/entity_declaration.py index 33ee5ff14..cd85991f5 100644 --- a/pyxform/entities/entity_declaration.py +++ b/pyxform/entities/entity_declaration.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from pyxform import constants as const from pyxform.survey_element import SurveyElement from pyxform.utils import node diff --git a/pyxform/errors.py b/pyxform/errors.py index 3de3e37e9..51aaf3845 100644 --- a/pyxform/errors.py +++ b/pyxform/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Common base classes for pyxform exceptions. """ diff --git a/pyxform/external_instance.py b/pyxform/external_instance.py index c2a5d322b..c90f06c20 100644 --- a/pyxform/external_instance.py +++ b/pyxform/external_instance.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ ExternalInstance class module """ diff --git a/pyxform/file_utils.py b/pyxform/file_utils.py index c827e6514..862ef115f 100644 --- a/pyxform/file_utils.py +++ b/pyxform/file_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ The pyxform file utility functions. """ diff --git a/pyxform/instance.py b/pyxform/instance.py index 8845b39a0..775458425 100644 --- a/pyxform/instance.py +++ b/pyxform/instance.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ SurveyInstance class module. """ diff --git a/pyxform/question.py b/pyxform/question.py index 4ddca1cb2..9019b95b1 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ XForm Survey element classes for different question types. """ diff --git a/pyxform/question_type_dictionary.py b/pyxform/question_type_dictionary.py index 43be6cdfe..84f35fbf5 100644 --- a/pyxform/question_type_dictionary.py +++ b/pyxform/question_type_dictionary.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ XForm survey question type mapping dictionary module. """ diff --git a/pyxform/section.py b/pyxform/section.py index ddc44a8d0..1fbd76fb8 100644 --- a/pyxform/section.py +++ b/pyxform/section.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Section survey element module. """ diff --git a/pyxform/survey.py b/pyxform/survey.py index af2d5e820..045e3438f 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Survey module with XForm Survey objects and utility functions. """ diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index c16222107..f4ccfbd2f 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Survey Element base class for all survey elements. """ diff --git a/pyxform/translator.py b/pyxform/translator.py index df79611af..e85564963 100644 --- a/pyxform/translator.py +++ b/pyxform/translator.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Translator class module. """ diff --git a/pyxform/utils.py b/pyxform/utils.py index 7d18e1bd9..837f9e5ec 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pyxform utils module. """ diff --git a/pyxform/validators/enketo_validate/__init__.py b/pyxform/validators/enketo_validate/__init__.py index 965005b77..50a7852ac 100644 --- a/pyxform/validators/enketo_validate/__init__.py +++ b/pyxform/validators/enketo_validate/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Validate XForms using Enketo validator. """ diff --git a/pyxform/validators/error_cleaner.py b/pyxform/validators/error_cleaner.py index 74a7d738d..f837f150b 100644 --- a/pyxform/validators/error_cleaner.py +++ b/pyxform/validators/error_cleaner.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Cleans up error messages from the validators. """ diff --git a/pyxform/validators/odk_validate/__init__.py b/pyxform/validators/odk_validate/__init__.py index fe9791498..f874bff9e 100644 --- a/pyxform/validators/odk_validate/__init__.py +++ b/pyxform/validators/odk_validate/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ odk_validate.py A python wrapper around ODK Validate diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index fa503ea04..708b7bb6b 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ pyxform_validator_update - command to update XForm validators. """ diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index 7e3b86dc4..f49e20e83 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ The validators utility functions. """ diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 118ee7202..b495a9a68 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ xform2json module - Transform an XForm to a JSON dictionary. """ diff --git a/pyxform/xform_instance_parser.py b/pyxform/xform_instance_parser.py index 6eca12dc5..bc2cfa360 100644 --- a/pyxform/xform_instance_parser.py +++ b/pyxform/xform_instance_parser.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ XFormInstanceParser class module - parses an instance XML. """ diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index de0f2b0e7..7c9dfed13 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ A Python script to convert excel files into JSON. """ diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 69039484e..4a1d01b03 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ XLS-to-dict and csv-to-dict are essentially backends for xls2json. """ diff --git a/pyxform/xls2xform.py b/pyxform/xls2xform.py index c47c80d38..190485bdc 100644 --- a/pyxform/xls2xform.py +++ b/pyxform/xls2xform.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ xls2xform converts properly formatted Excel documents into XForms for use with ODK Collect. diff --git a/tests/builder_tests.py b/tests/builder_tests.py index bbadeeba2..a230d46ea 100644 --- a/tests/builder_tests.py +++ b/tests/builder_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test builder module functionality. """ diff --git a/tests/dump_and_load_tests.py b/tests/dump_and_load_tests.py index 6c0169b25..5ddb2d035 100644 --- a/tests/dump_and_load_tests.py +++ b/tests/dump_and_load_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test multiple XLSForm can be generated successfully. """ diff --git a/tests/file_utils_test.py b/tests/file_utils_test.py index cb8bd9b3c..df50d123e 100644 --- a/tests/file_utils_test.py +++ b/tests/file_utils_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xls2json_backends util functions. """ diff --git a/tests/group_test.py b/tests/group_test.py index 2455ece67..68597cc76 100644 --- a/tests/group_test.py +++ b/tests/group_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing simple cases for Xls2Json """ diff --git a/tests/j2x_question_tests.py b/tests/j2x_question_tests.py index 2bcbcd240..7ba69f9ec 100644 --- a/tests/j2x_question_tests.py +++ b/tests/j2x_question_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing creation of Surveys using verbose methods """ diff --git a/tests/j2x_test_creation.py b/tests/j2x_test_creation.py index af0a303fd..87218d84f 100644 --- a/tests/j2x_test_creation.py +++ b/tests/j2x_test_creation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing creation of Surveys using verbose methods """ diff --git a/tests/j2x_test_instantiation.py b/tests/j2x_test_instantiation.py index d4c1f5736..77e8e8a3d 100644 --- a/tests/j2x_test_instantiation.py +++ b/tests/j2x_test_instantiation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing the instance object for pyxform. """ diff --git a/tests/j2x_test_xform_build_preparation.py b/tests/j2x_test_xform_build_preparation.py index f3b4ca5b6..3740c908b 100644 --- a/tests/j2x_test_xform_build_preparation.py +++ b/tests/j2x_test_xform_build_preparation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing preparation of values for XForm exporting """ diff --git a/tests/js2x_test_import_from_json.py b/tests/js2x_test_import_from_json.py index d3e0ba631..7a698c8dd 100644 --- a/tests/js2x_test_import_from_json.py +++ b/tests/js2x_test_import_from_json.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing our ability to import from a JSON text file. """ diff --git a/tests/json2xform_test.py b/tests/json2xform_test.py index bab2d9040..c169debd3 100644 --- a/tests/json2xform_test.py +++ b/tests/json2xform_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing simple cases for pyxform """ diff --git a/tests/loop_tests.py b/tests/loop_tests.py index e6a52a609..d062a6b9c 100644 --- a/tests/loop_tests.py +++ b/tests/loop_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test loop syntax. """ diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 4738233f8..95b81588c 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ PyxformTestCase base class using markdown to define the XLSForm. """ diff --git a/tests/settings_test.py b/tests/settings_test.py index 4bf2289a3..de52795ee 100644 --- a/tests/settings_test.py +++ b/tests/settings_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test settings sheet syntax. """ diff --git a/tests/test_allow_mock_accuracy.py b/tests/test_allow_mock_accuracy.py index e5952040c..bb58c821b 100644 --- a/tests/test_allow_mock_accuracy.py +++ b/tests/test_allow_mock_accuracy.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_area.py b/tests/test_area.py index d6483c74a..10cf6360f 100644 --- a/tests/test_area.py +++ b/tests/test_area.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ AreaTest - test enclosed-area(geo_shape) calculation. """ diff --git a/tests/test_audio_quality.py b/tests/test_audio_quality.py index 36f28f338..02ef21b3a 100644 --- a/tests/test_audio_quality.py +++ b/tests/test_audio_quality.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_audit.py b/tests/test_audit.py index 36b5f19f6..6ef65a292 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ AuditTest - test audit question type. """ diff --git a/tests/test_background_audio.py b/tests/test_background_audio.py index 9795657b3..ca4a6e574 100644 --- a/tests/test_background_audio.py +++ b/tests/test_background_audio.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_bind_conversions.py b/tests/test_bind_conversions.py index 70581be81..d6d58016c 100644 --- a/tests/test_bind_conversions.py +++ b/tests/test_bind_conversions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ BindConversionsTest - test bind conversions. """ diff --git a/tests/test_bug_missing_headers.py b/tests/test_bug_missing_headers.py index 8fdaf58f5..7013d8719 100644 --- a/tests/test_bug_missing_headers.py +++ b/tests/test_bug_missing_headers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test missing headers in XLSForm. """ diff --git a/tests/test_bug_round_calculation.py b/tests/test_bug_round_calculation.py index ffcf6418c..64f52e352 100644 --- a/tests/test_bug_round_calculation.py +++ b/tests/test_bug_round_calculation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test round(number, precision) calculation. """ diff --git a/tests/test_choices_sheet.py b/tests/test_choices_sheet.py index e230ca1c5..114ccfc35 100644 --- a/tests/test_choices_sheet.py +++ b/tests/test_choices_sheet.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase from tests.xpath_helpers.choices import xpc from tests.xpath_helpers.questions import xpq diff --git a/tests/test_custom_xml_namespaces.py b/tests/test_custom_xml_namespaces.py index 09bf9d4f0..57933eb5f 100644 --- a/tests/test_custom_xml_namespaces.py +++ b/tests/test_custom_xml_namespaces.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test custom namespaces. """ diff --git a/tests/test_dynamic_default.py b/tests/test_dynamic_default.py index ad8c6f0b1..0ea4ad4f5 100644 --- a/tests/test_dynamic_default.py +++ b/tests/test_dynamic_default.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test handling dynamic default in forms """ diff --git a/tests/test_entities_create.py b/tests/test_entities_create.py index bd3d9055e..2a39b656e 100644 --- a/tests/test_entities_create.py +++ b/tests/test_entities_create.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_entities_update.py b/tests/test_entities_update.py index 5e326607b..9ec7616e9 100644 --- a/tests/test_entities_update.py +++ b/tests/test_entities_update.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_external_instances.py b/tests/test_external_instances.py index b6f89b776..1f072d0c9 100644 --- a/tests/test_external_instances.py +++ b/tests/test_external_instances.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xml-external syntax and instances generated from pulldata calls. diff --git a/tests/test_external_instances_for_selects.py b/tests/test_external_instances_for_selects.py index 048b50318..73e67d3e8 100644 --- a/tests/test_external_instances_for_selects.py +++ b/tests/test_external_instances_for_selects.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test external instance syntax diff --git a/tests/test_fieldlist_labels.py b/tests/test_fieldlist_labels.py index 448e05a90..bf3a4badc 100644 --- a/tests/test_fieldlist_labels.py +++ b/tests/test_fieldlist_labels.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test field-list labels """ diff --git a/tests/test_fields.py b/tests/test_fields.py index 61f001961..86afb9c7c 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test duplicate survey question field name. """ diff --git a/tests/test_file.py b/tests/test_file.py index 67c7f3678..21970c010 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test file question type. """ diff --git a/tests/test_for_loop.py b/tests/test_for_loop.py index a068f12d0..03f7d2585 100644 --- a/tests/test_for_loop.py +++ b/tests/test_for_loop.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test loop question type. """ diff --git a/tests/test_form_name.py b/tests/test_form_name.py index 79dd78f0f..11a053003 100644 --- a/tests/test_form_name.py +++ b/tests/test_form_name.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test setting form name to data. """ diff --git a/tests/test_geo.py b/tests/test_geo.py index 00f412b9f..548295e54 100644 --- a/tests/test_geo.py +++ b/tests/test_geo.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test geo widgets. """ diff --git a/tests/test_groups.py b/tests/test_groups.py index 21b223aea..5853fcfee 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XForm groups. """ diff --git a/tests/test_guidance_hint.py b/tests/test_guidance_hint.py index 90e063cec..8566a9c48 100644 --- a/tests/test_guidance_hint.py +++ b/tests/test_guidance_hint.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Guidance hint test module. """ diff --git a/tests/test_image_app_parameter.py b/tests/test_image_app_parameter.py index 6f6324f5d..36f55bab5 100644 --- a/tests/test_image_app_parameter.py +++ b/tests/test_image_app_parameter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test image max-pixels and app parameters. """ diff --git a/tests/test_language_warnings.py b/tests/test_language_warnings.py index 5171d9e51..2d17a9a75 100644 --- a/tests/test_language_warnings.py +++ b/tests/test_language_warnings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test language warnings. """ diff --git a/tests/test_last_saved.py b/tests/test_last_saved.py index 5c717c9ef..05f771a02 100644 --- a/tests/test_last_saved.py +++ b/tests/test_last_saved.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ The last-saved virtual instance can be queried to get values from the last saved instance of the form being authored. """ diff --git a/tests/test_metadata.py b/tests/test_metadata.py index cc9041f5e..0fcd1a8e3 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test language warnings. """ diff --git a/tests/test_osm.py b/tests/test_osm.py index 5f9d3c390..1905a4df2 100644 --- a/tests/test_osm.py +++ b/tests/test_osm.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test OSM widgets. """ diff --git a/tests/test_parameters_rows.py b/tests/test_parameters_rows.py index d857d83ed..035556083 100644 --- a/tests/test_parameters_rows.py +++ b/tests/test_parameters_rows.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test text rows parameter. """ diff --git a/tests/test_pyxformtestcase.py b/tests/test_pyxformtestcase.py index e0b1acaa4..35a35a02b 100644 --- a/tests/test_pyxformtestcase.py +++ b/tests/test_pyxformtestcase.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Ensuring that the pyxform_test_case.PyxformTestCase class does some internal conversions correctly. diff --git a/tests/test_randomize_itemsets.py b/tests/test_randomize_itemsets.py index 6a5d90b03..6496e441e 100644 --- a/tests/test_randomize_itemsets.py +++ b/tests/test_randomize_itemsets.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test randomize itemsets. """ diff --git a/tests/test_range.py b/tests/test_range.py index dde26bb10..14bb5553e 100644 --- a/tests/test_range.py +++ b/tests/test_range.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test range widget. """ diff --git a/tests/test_rank.py b/tests/test_rank.py index 22d0001b8..f41386e4d 100644 --- a/tests/test_rank.py +++ b/tests/test_rank.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test rank widget. """ diff --git a/tests/test_repeat.py b/tests/test_repeat.py index 655ea474e..4bb0cb999 100644 --- a/tests/test_repeat.py +++ b/tests/test_repeat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test reapeat structure. """ diff --git a/tests/test_repeat_template.py b/tests/test_repeat_template.py index 3f8ed79cc..155952162 100644 --- a/tests/test_repeat_template.py +++ b/tests/test_repeat_template.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test repeat template and instance structure. """ diff --git a/tests/test_secondary_instance_translations.py b/tests/test_secondary_instance_translations.py index 417e63ed3..bd80f709f 100644 --- a/tests/test_secondary_instance_translations.py +++ b/tests/test_secondary_instance_translations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing inlining translation when no translation is specified. """ diff --git a/tests/test_set_geopoint.py b/tests/test_set_geopoint.py index d897d063c..0540cbbfa 100644 --- a/tests/test_set_geopoint.py +++ b/tests/test_set_geopoint.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test setgeopoint widget. """ diff --git a/tests/test_settings_auto_send_delete.py b/tests/test_settings_auto_send_delete.py index 0b1e302d6..ac8df5af3 100644 --- a/tests/test_settings_auto_send_delete.py +++ b/tests/test_settings_auto_send_delete.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test settins auto settings. """ diff --git a/tests/test_sheet_columns.py b/tests/test_sheet_columns.py index 9b730b0f3..5b60c5498 100644 --- a/tests/test_sheet_columns.py +++ b/tests/test_sheet_columns.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XLSForm sheet names. """ diff --git a/tests/test_sms.py b/tests/test_sms.py index cf7e55b77..005780716 100644 --- a/tests/test_sms.py +++ b/tests/test_sms.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test sms syntax. """ diff --git a/tests/test_table_list.py b/tests/test_table_list.py index a61409211..155c157c4 100644 --- a/tests/test_table_list.py +++ b/tests/test_table_list.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test table list appearance syntax. """ diff --git a/tests/test_translations.py b/tests/test_translations.py index c5b0bfd2b..bc460bea6 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test translations syntax. """ diff --git a/tests/test_trigger.py b/tests/test_trigger.py index e8002a37d..b3cdb6727 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test handling setvalue of 'trigger' column in forms """ diff --git a/tests/test_typed_calculates.py b/tests/test_typed_calculates.py index 396afef80..5a13f3f77 100644 --- a/tests/test_typed_calculates.py +++ b/tests/test_typed_calculates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test that any row with a calculation becomes a calculate of the row's type or of type string if the type is "calculate". A hint or label error should only be thrown for a row without a calculation. diff --git a/tests/test_unicode_rtl.py b/tests/test_unicode_rtl.py index 130b6949b..ec28ad80a 100644 --- a/tests/test_unicode_rtl.py +++ b/tests/test_unicode_rtl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test unicode rtl in XLSForms. """ diff --git a/tests/test_utils/md_table.py b/tests/test_utils/md_table.py index b1aafe17f..9903c417a 100644 --- a/tests/test_utils/md_table.py +++ b/tests/test_utils/md_table.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Markdown table utility functions. """ diff --git a/tests/test_validate_unicode_exception.py b/tests/test_validate_unicode_exception.py index fa93392dd..4648f0cd4 100644 --- a/tests/test_validate_unicode_exception.py +++ b/tests/test_validate_unicode_exception.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test unicode characters in validate error messages. """ diff --git a/tests/test_validator_update.py b/tests/test_validator_update.py index 4deb2f88e..9f7e3cb94 100644 --- a/tests/test_validator_update.py +++ b/tests/test_validator_update.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test validator update cli command. """ diff --git a/tests/test_validator_util.py b/tests/test_validator_util.py index e39a75141..147bf728a 100644 --- a/tests/test_validator_util.py +++ b/tests/test_validator_util.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test pyxform.validators.utils module. """ diff --git a/tests/test_validators.py b/tests/test_validators.py index 39e2b48b0..0d5a08e09 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test validators. """ diff --git a/tests/test_warnings.py b/tests/test_warnings.py index f8b4ab693..2f8a38729 100644 --- a/tests/test_warnings.py +++ b/tests/test_warnings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test warnings. """ diff --git a/tests/test_whitespace.py b/tests/test_whitespace.py index 2f19615ac..9e3c9c01f 100644 --- a/tests/test_whitespace.py +++ b/tests/test_whitespace.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test whitespace around output variables in XForms. """ diff --git a/tests/test_xform2json.py b/tests/test_xform2json.py index 061190fe2..3980388c1 100644 --- a/tests/test_xform2json.py +++ b/tests/test_xform2json.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XForm2JSON functionality """ diff --git a/tests/test_xls2json_backends.py b/tests/test_xls2json_backends.py index d3d17e488..55a9fcfcf 100644 --- a/tests/test_xls2json_backends.py +++ b/tests/test_xls2json_backends.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xls2json_backends module functionality. """ diff --git a/tests/test_xlsform_headers.py b/tests/test_xlsform_headers.py index c588b72a0..b1fa34222 100644 --- a/tests/test_xlsform_headers.py +++ b/tests/test_xlsform_headers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XLSForm headers syntax. """ diff --git a/tests/test_xml_structure.py b/tests/test_xml_structure.py index 120a6e2a2..6be5e6442 100644 --- a/tests/test_xml_structure.py +++ b/tests/test_xml_structure.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XForm structure. """ diff --git a/tests/tutorial_test.py b/tests/tutorial_test.py index 7a9dd5de5..57b15d537 100644 --- a/tests/tutorial_test.py +++ b/tests/tutorial_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test tutorial XLSForm. """ diff --git a/tests/utils.py b/tests/utils.py index e2828c3cf..e68991fa2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ The tests utils module functionality. """ diff --git a/tests/validators/__init__.py b/tests/validators/__init__.py index abbc7fd26..1b011eb96 100644 --- a/tests/validators/__init__.py +++ b/tests/validators/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os HERE = os.path.dirname(__file__) diff --git a/tests/validators/server.py b/tests/validators/server.py index 0847817e5..4d1f4ee1b 100644 --- a/tests/validators/server.py +++ b/tests/validators/server.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os import posixpath import threading diff --git a/tests/xform2json_test.py b/tests/xform2json_test.py index ba2ad8e0b..f2e7c4ff7 100644 --- a/tests/xform2json_test.py +++ b/tests/xform2json_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xform2json module. """ diff --git a/tests/xform_test_case/attributecolumnstest.py b/tests/xform_test_case/attributecolumnstest.py index 59fda3b50..9f4500d3b 100644 --- a/tests/xform_test_case/attributecolumnstest.py +++ b/tests/xform_test_case/attributecolumnstest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Some tests for the new (v0.9) spec is properly implemented. """ diff --git a/tests/xform_test_case/bug_tests.py b/tests/xform_test_case/bug_tests.py index d2d636efa..c7312c770 100644 --- a/tests/xform_test_case/bug_tests.py +++ b/tests/xform_test_case/bug_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Some tests for the new (v0.9) spec is properly implemented. """ diff --git a/tests/xform_test_case/xlsform_spec_test.py b/tests/xform_test_case/xlsform_spec_test.py index a1c9dd585..b21ff08c1 100644 --- a/tests/xform_test_case/xlsform_spec_test.py +++ b/tests/xform_test_case/xlsform_spec_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Some tests for the new (v0.9) spec is properly implemented. """ diff --git a/tests/xform_test_case/xml_tests.py b/tests/xform_test_case/xml_tests.py index 311d286e2..cfeb4271b 100644 --- a/tests/xform_test_case/xml_tests.py +++ b/tests/xform_test_case/xml_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XForm XML syntax. """ diff --git a/tests/xls2json_tests.py b/tests/xls2json_tests.py index 4a440a763..564ba4091 100644 --- a/tests/xls2json_tests.py +++ b/tests/xls2json_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing simple cases for Xls2Json """ diff --git a/tests/xls2xform_tests.py b/tests/xls2xform_tests.py index 4a65f7f60..7329d2738 100644 --- a/tests/xls2xform_tests.py +++ b/tests/xls2xform_tests.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xls2xform module. """ From 68da45f4580b476adfecdca8a3d27269f64ed30b Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 03:53:03 +1100 Subject: [PATCH 11/22] dev: fix linter warnings from pyupgrade rules --- pyproject.toml | 9 +++++++++ pyxform/question.py | 6 +++--- pyxform/section.py | 9 ++++----- pyxform/survey.py | 4 ++-- pyxform/survey_element.py | 3 +-- pyxform/utils.py | 5 ++--- pyxform/validators/enketo_validate/__init__.py | 2 +- pyxform/validators/odk_validate/__init__.py | 2 +- pyxform/validators/updater.py | 13 ++++++------- pyxform/validators/util.py | 9 ++++----- pyxform/xform2json.py | 2 +- pyxform/xform_instance_parser.py | 5 ++--- pyxform/xls2json.py | 2 +- pyxform/xls2xform.py | 2 +- tests/j2x_test_creation.py | 2 +- tests/j2x_test_instantiation.py | 8 ++++---- tests/loop_tests.py | 2 +- tests/test_form_name.py | 2 +- tests/validators/server.py | 4 ++-- tests/xform_test_case/bug_tests.py | 2 +- 20 files changed, 48 insertions(+), 45 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f9120c8a2..5b57e4081 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,16 @@ select = [ # "SIM", # flake8-simplify "TRY", # tryceratops # "UP", # pyupgrade + "UP004", + "UP005", + "UP008", "UP009", + "UP018", + "UP020", + "UP024", + "UP028", + "UP030", + "UP035", "W", # pycodestyle warning ] ignore = [ diff --git a/pyxform/question.py b/pyxform/question.py index 9019b95b1..2b2a5fb70 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -305,7 +305,7 @@ def build_xml(self): class SelectOneQuestion(MultipleChoiceQuestion): def __init__(self, **kwargs): - super(SelectOneQuestion, self).__init__(**kwargs) + super().__init__(**kwargs) self._dict[self.TYPE] = "select one" @@ -314,7 +314,7 @@ def __init__(self, **kwargs): kwargs_copy = kwargs.copy() choices = kwargs_copy.pop("choices", []) + kwargs_copy.pop("children", []) - super(Tag, self).__init__(**kwargs_copy) + super().__init__(**kwargs_copy) if choices: self.children = [] @@ -344,7 +344,7 @@ def __init__(self, **kwargs): kwargs_copy = kwargs.copy() tags = kwargs_copy.pop("tags", []) + kwargs_copy.pop("children", []) - super(OsmUploadQuestion, self).__init__(**kwargs_copy) + super().__init__(**kwargs_copy) if tags: self.children = [] diff --git a/pyxform/section.py b/pyxform/section.py index 1fbd76fb8..54d6c0c85 100644 --- a/pyxform/section.py +++ b/pyxform/section.py @@ -9,7 +9,7 @@ class Section(SurveyElement): def validate(self): - super(Section, self).validate() + super().validate() for element in self.children: element.validate() self._validate_uniqueness_of_element_names() @@ -76,8 +76,7 @@ def xml_instance_array(self): """ for child in self.children: if child.get("flat"): - for grandchild in child.xml_instance_array(): - yield grandchild + yield from child.xml_instance_array() else: yield child.xml_instance() @@ -144,7 +143,7 @@ def _dynamic_defaults_helper(self, current, nodes): # I'm anal about matching function signatures when overriding a function, # but there's no reason for kwargs to be an argument def template_instance(self, **kwargs): - return super(RepeatingSection, self).generate_repeating_template(**kwargs) + return super().generate_repeating_template(**kwargs) class GroupedSection(Section): @@ -196,6 +195,6 @@ def xml_control(self): def to_json_dict(self): # This is quite hacky, might want to think about a smart way # to approach this problem. - result = super(GroupedSection, self).to_json_dict() + result = super().to_json_dict() result["type"] = "group" return result diff --git a/pyxform/survey.py b/pyxform/survey.py index 045e3438f..18fbb9890 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -222,7 +222,7 @@ class Survey(Section): def validate(self): if self.id_string in [None, "None"]: raise PyXFormError("Survey cannot have an empty id_string") - super(Survey, self).validate() + super().validate() self._validate_uniqueness_of_section_names() def _validate_uniqueness_of_section_names(self): @@ -1059,7 +1059,7 @@ def _is_return_relative_path() -> bool: .group(1) .split(",") ) - name_arg = "${{{0}}}".format(name) + name_arg = "${{{}}}".format(name) for idx, arg in enumerate(indexed_repeat_args): if name_arg in arg.strip(): indexed_repeat_name_index = idx diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index f4ccfbd2f..2076e2688 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -155,8 +155,7 @@ def iter_descendants(self): # it really seems like this method should not yield self yield self for e in self.children: - for f in e.iter_descendants(): - yield f + yield from e.iter_descendants() def any_repeat(self, parent_xpath: str) -> bool: """Return True if there ia any repeat in `parent_xpath`.""" diff --git a/pyxform/utils.py b/pyxform/utils.py index 837f9e5ec..cccc93204 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -167,7 +167,7 @@ def get_pyobj_from_json(str_or_path): # see if treating str_or_path as a path works fp = codecs.open(str_or_path, mode="r", encoding="utf-8") doc = json.load(fp) - except (IOError, JSONDecodeError, OSError): + except (JSONDecodeError, OSError): # if it doesn't work load the text doc = json.loads(str_or_path) return doc @@ -175,8 +175,7 @@ def get_pyobj_from_json(str_or_path): def flatten(li): for subli in li: - for it in subli: - yield it + yield from subli def sheet_to_csv(workbook_path, csv_path, sheet_name): diff --git a/pyxform/validators/enketo_validate/__init__.py b/pyxform/validators/enketo_validate/__init__.py index 50a7852ac..ef2935dad 100644 --- a/pyxform/validators/enketo_validate/__init__.py +++ b/pyxform/validators/enketo_validate/__init__.py @@ -60,7 +60,7 @@ def check_xform(path_to_xform): :return: warnings or List[str] """ if not install_exists(): - raise EnvironmentError( + raise OSError( "Enketo-validate dependency not found. " "Please use the updater tool to install the latest version." ) diff --git a/pyxform/validators/odk_validate/__init__.py b/pyxform/validators/odk_validate/__init__.py index f874bff9e..9616fafb4 100644 --- a/pyxform/validators/odk_validate/__init__.py +++ b/pyxform/validators/odk_validate/__init__.py @@ -65,7 +65,7 @@ def check_java_available(): "To fix this, please either: 1) install Java, or 2) run pyxform with the " "--skip_validate flag, or 3) add the installed Java to the environment path." ) - raise EnvironmentError(msg) + raise OSError(msg) def check_xform(path_to_xform): diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index 708b7bb6b..b9281a4ff 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -3,7 +3,6 @@ """ import argparse import fnmatch -import io import json import logging import os @@ -97,7 +96,7 @@ def _read_json(file_path): Read the JSON file to a string. """ _UpdateHandler._check_path(file_path=file_path) - with io.open(file_path, mode="r") as in_file: + with open(file_path, mode="r") as in_file: return json.load(in_file) @staticmethod @@ -105,7 +104,7 @@ def _write_json(file_path, content): """ Save the JSON data to a file. """ - with io.open(file_path, mode="w", newline="\n") as out_file: + with open(file_path, mode="w", newline="\n") as out_file: data = json.dumps(content, indent=2, sort_keys=True) out_file.write(str(data)) @@ -115,7 +114,7 @@ def _read_last_check(file_path): Read the .last_check file. """ _UpdateHandler._check_path(file_path=file_path) - with io.open(file_path, mode="r") as in_file: + with open(file_path, mode="r") as in_file: first_line = in_file.readline() try: last_check = datetime.strptime(first_line, UTC_FMT) @@ -129,7 +128,7 @@ def _write_last_check(file_path, content): """ Write the .last_check file. """ - with io.open(file_path, mode="w", newline="\n") as out_file: + with open(file_path, mode="w", newline="\n") as out_file: out_file.write(str(content.strftime(UTC_FMT))) @staticmethod @@ -248,7 +247,7 @@ def _download_file(url, file_path): """ Save response content from the URL to a binary file at the file path. """ - with io.open(file_path, mode="wb") as out_file: + with open(file_path, mode="wb") as out_file: file_data = request_get(url=url) out_file.write(file_data) @@ -330,7 +329,7 @@ def _unzip_extract_file(open_zip_file, zip_item, file_out_path): os.makedirs(out_parent) with open_zip_file.open(zip_item, mode="r") as zip_item_file: zip_item_data = zip_item_file.read() - with io.open(file_out_path, "wb") as file_out_file: + with open(file_out_path, "wb") as file_out_file: file_out_file.write(zip_item_data) @staticmethod diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index f49e20e83..b38d494b1 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -1,7 +1,6 @@ """ The validators utility functions. """ -import io import logging import os import signal @@ -99,7 +98,7 @@ def decode_stream(stream): return stream.decode("latin-1") except BaseException as be: msg = "Failed to decode validate stderr as utf-8 or latin-1." - raise IOError(msg, ude, be) from be + raise OSError(msg, ude, be) from be def request_get(url): @@ -196,9 +195,9 @@ def check_readable(file_path, retry_limit=10, wait_seconds=0.5): def catch_try(): try: - with io.open(file_path, mode="r"): + with open(file_path, mode="r"): return True - except IOError: + except OSError: return False tries = 0 @@ -207,5 +206,5 @@ def catch_try(): tries += 1 time.sleep(wait_seconds) else: - raise IOError("Could not read file: {f}".format(f=file_path)) + raise OSError("Could not read file: {f}".format(f=file_path)) return True diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index b495a9a68..f93e50462 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -6,7 +6,7 @@ import json import logging import re -from collections import Mapping +from collections.abc import Mapping from operator import itemgetter from typing import Any, Dict, List from xml.etree.ElementTree import Element diff --git a/pyxform/xform_instance_parser.py b/pyxform/xform_instance_parser.py index bc2cfa360..7c41dc1de 100644 --- a/pyxform/xform_instance_parser.py +++ b/pyxform/xform_instance_parser.py @@ -82,8 +82,7 @@ def _get_all_attributes(node): for key in node.attributes.keys(): yield key, node.getAttribute(key) for child in node.childNodes: - for pair in _get_all_attributes(child): - yield pair + yield from _get_all_attributes(child) class XFormInstanceParser: @@ -92,7 +91,7 @@ def __init__(self, xml_str): def parse(self, xml_str): clean_xml_str = xml_str.strip() - clean_xml_str = re.sub(str(r">\s+<"), str("><"), clean_xml_str) + clean_xml_str = re.sub(r">\s+<", "><", clean_xml_str) self._xml_obj = parseString(clean_xml_str) self._root_node = self._xml_obj.documentElement self._dict = _xml_node_to_dict(self._root_node) diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 7c9dfed13..74c2a1288 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -1698,7 +1698,7 @@ class QuestionTypesReader(SpreadsheetReader): """ def __init__(self, path): - super(QuestionTypesReader, self).__init__(path) + super().__init__(path) self._setup_question_types_dictionary() def _setup_question_types_dictionary(self): diff --git a/pyxform/xls2xform.py b/pyxform/xls2xform.py index 190485bdc..e31d5e1af 100644 --- a/pyxform/xls2xform.py +++ b/pyxform/xls2xform.py @@ -173,7 +173,7 @@ def main_cli(): pretty_print=args.pretty_print, enketo=args.enketo_validate, ) - except EnvironmentError: + except OSError: # Do not crash if 'java' not installed logger.exception("EnvironmentError during conversion") except ODKValidateError: diff --git a/tests/j2x_test_creation.py b/tests/j2x_test_creation.py index 87218d84f..e8f79ad3c 100644 --- a/tests/j2x_test_creation.py +++ b/tests/j2x_test_creation.py @@ -89,4 +89,4 @@ def allow_surveys_with_comment_rows(self): }, "type": "survey", } - self.assertEquals(survey.to_json_dict(), expected_dict) + self.assertEqual(survey.to_json_dict(), expected_dict) diff --git a/tests/j2x_test_instantiation.py b/tests/j2x_test_instantiation.py index 77e8e8a3d..0b419be14 100644 --- a/tests/j2x_test_instantiation.py +++ b/tests/j2x_test_instantiation.py @@ -26,8 +26,8 @@ def test_simple_survey_instantiation(self): i = surv.instantiate() - self.assertEquals(i.keys(), ["survey_question"]) - self.assertEquals(set(i.xpaths()), {"/Simple", "/Simple/survey_question"}) + self.assertEqual(i.keys(), ["survey_question"]) + self.assertEqual(set(i.xpaths()), {"/Simple", "/Simple/survey_question"}) def test_simple_survey_answering(self): surv = Survey(name="Water") @@ -43,10 +43,10 @@ def test_simple_survey_answering(self): i = SurveyInstance(surv) i.answer(name="color", value="blue") - self.assertEquals(i.answers()["color"], "blue") + self.assertEqual(i.answers()["color"], "blue") i.answer(name="feeling", value="liquidy") - self.assertEquals(i.answers()["feeling"], "liquidy") + self.assertEqual(i.answers()["feeling"], "liquidy") def test_answers_can_be_imported_from_xml(self): surv = Survey(name="data") diff --git a/tests/loop_tests.py b/tests/loop_tests.py index d062a6b9c..5a0799500 100644 --- a/tests/loop_tests.py +++ b/tests/loop_tests.py @@ -89,4 +89,4 @@ def test_loop(self): }, ], } - self.assertEquals(survey.to_json_dict(), expected_dict) + self.assertEqual(survey.to_json_dict(), expected_dict) diff --git a/tests/test_form_name.py b/tests/test_form_name.py index 11a053003..36df4edb0 100644 --- a/tests/test_form_name.py +++ b/tests/test_form_name.py @@ -23,7 +23,7 @@ def test_default_to_data_when_no_name(self): # We're passing autoname false when creating the survey object. self.assertEqual(survey.id_string, None) - self.assertEqual(survey.name, str("data")) + self.assertEqual(survey.name, "data") self.assertEqual(survey.title, None) # Set required fields because we need them if we want to do xml comparison. diff --git a/tests/validators/server.py b/tests/validators/server.py index 4d1f4ee1b..52d579dd3 100644 --- a/tests/validators/server.py +++ b/tests/validators/server.py @@ -9,7 +9,7 @@ HERE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") -class SimpleHTTPRequestHandlerHere(SimpleHTTPRequestHandler, object): +class SimpleHTTPRequestHandlerHere(SimpleHTTPRequestHandler): def send_head(self): if self.client_address[0] != "127.0.0.1": self.send_error( @@ -17,7 +17,7 @@ def send_head(self): ) return None else: - return super(SimpleHTTPRequestHandlerHere, self).send_head() + return super().send_head() def translate_path(self, path): """ diff --git a/tests/xform_test_case/bug_tests.py b/tests/xform_test_case/bug_tests.py index c7312c770..b88c1d162 100644 --- a/tests/xform_test_case/bug_tests.py +++ b/tests/xform_test_case/bug_tests.py @@ -188,7 +188,7 @@ def runTest(self): default_name="spaces_in_choices_header", warnings=warnings, ) - self.assertEquals(len(warnings), 3, "Found " + str(len(warnings)) + " warnings") + self.assertEqual(len(warnings), 3, "Found " + str(len(warnings)) + " warnings") def test_values_with_spaces_are_cleaned(self): """ From f77b252deb14e83d46b2f235c79fc853a8070f35 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 03:58:20 +1100 Subject: [PATCH 12/22] dev: fix linter warnings from pyupgrade rules --- pyproject.toml | 3 +++ pyxform/entities/entities_parsing.py | 2 +- pyxform/survey.py | 25 ++++++------------- .../validators/pyxform/select_from_file.py | 2 +- pyxform/validators/updater.py | 18 ++++++------- pyxform/validators/util.py | 16 ++++++------ pyxform/xform2json.py | 2 +- pyxform/xls2json.py | 9 +++---- pyxform/xls2json_backends.py | 2 +- pyxform/xlsparseutils.py | 2 +- tests/test_dynamic_default.py | 4 +-- tests/test_external_instances_for_selects.py | 2 +- tests/test_survey.py | 8 +++--- tests/test_translations.py | 4 +-- tests/test_xls2json.py | 18 ++++++------- tests/xform_test_case/xlsform_spec_test.py | 4 +-- 16 files changed, 55 insertions(+), 66 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5b57e4081..17733dffa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,11 +77,14 @@ select = [ "UP005", "UP008", "UP009", + "UP015", "UP018", "UP020", "UP024", "UP028", "UP030", + "UP032", + "UP034", "UP035", "W", # pycodestyle warning ] diff --git a/pyxform/entities/entities_parsing.py b/pyxform/entities/entities_parsing.py index 85a758441..948d61db8 100644 --- a/pyxform/entities/entities_parsing.py +++ b/pyxform/entities/entities_parsing.py @@ -127,7 +127,7 @@ def validate_entity_saveto( def validate_entities_columns(row: Dict): extra = {k: None for k in row.keys() if k not in EC.value_list()} if 0 < len(extra): - fmt_extra = ", ".join((f"'{k}'" for k in extra.keys())) + fmt_extra = ", ".join(f"'{k}'" for k in extra.keys()) msg = ( f"The entities sheet included the following unexpected column(s): {fmt_extra}. " f"These columns are not supported by this version of pyxform. Please either: " diff --git a/pyxform/survey.py b/pyxform/survey.py index 18fbb9890..915676eba 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -355,7 +355,7 @@ def _generate_external_instances(element) -> Optional[InstanceInfo]: name = element["name"] extension = element["type"].split("-")[0] prefix = "file-csv" if extension == "csv" else "file" - src = "jr://{}/{}.{}".format(prefix, name, extension) + src = f"jr://{prefix}/{name}.{extension}" return InstanceInfo( type="external", context="[type: {t}, name: {n}]".format( @@ -389,10 +389,8 @@ def _validate_external_instances(instances) -> None: contexts = ", ".join(x.context for x in copies) errors.append( "Instance names must be unique within a form. " - "The name '{i}' was found {c} time(s), " - "under these contexts: {contexts}".format( - i=element, c=len(copies), contexts=contexts - ) + f"The name '{element}' was found {len(copies)} time(s), " + f"under these contexts: {contexts}" ) if errors: raise ValidationError("\n".join(errors)) @@ -419,7 +417,7 @@ def get_pulldata_functions(element): return functions_present def get_instance_info(element, file_id): - uri = "jr://file-csv/{}.csv".format(file_id) + uri = f"jr://file-csv/{file_id}.csv" return InstanceInfo( type="pulldata", @@ -566,16 +564,9 @@ def _generate_instances(self) -> Iterator[DetachableElement]: msg = ( "The same instance id will be generated for different " "external instance source URIs. Please check the form." - " Instance name: '{i}', Existing type: '{e}', " - "Existing URI: '{iu}', Duplicate type: '{d}', " - "Duplicate URI: '{du}', Duplicate context: '{c}'.".format( - i=i.name, - iu=seen[i.name].src, - e=seen[i.name].type, - d=i.type, - du=i.src, - c=i.context, - ) + f" Instance name: '{i.name}', Existing type: '{seen[i.name].type}', " + f"Existing URI: '{seen[i.name].src}', Duplicate type: '{i.type}', " + f"Duplicate URI: '{i.src}', Duplicate context: '{i.context}'." ) raise PyXFormError(msg) elif i.name in seen.keys() and seen[i.name].src == i.src: @@ -1059,7 +1050,7 @@ def _is_return_relative_path() -> bool: .group(1) .split(",") ) - name_arg = "${{{}}}".format(name) + name_arg = f"${{{name}}}" for idx, arg in enumerate(indexed_repeat_args): if name_arg in arg.strip(): indexed_repeat_name_index = idx diff --git a/pyxform/validators/pyxform/select_from_file.py b/pyxform/validators/pyxform/select_from_file.py index be29be70c..33ace1cce 100644 --- a/pyxform/validators/pyxform/select_from_file.py +++ b/pyxform/validators/pyxform/select_from_file.py @@ -54,7 +54,7 @@ def validate_list_name_extension( 1 != len(list_path.suffixes) or list_path.suffix not in EXTERNAL_INSTANCE_EXTENSIONS ): - exts = ", ".join((f"'{e}'" for e in EXTERNAL_INSTANCE_EXTENSIONS)) + exts = ", ".join(f"'{e}'" for e in EXTERNAL_INSTANCE_EXTENSIONS) raise PyXFormError( ROW_FORMAT_STRING % row_number + f" File name for '{select_command} {list_name}' should end with one of " diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index b9281a4ff..9b5fea5b0 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -62,7 +62,7 @@ def __init__( self.installed_path = os.path.join(self.bin_path, "installed.json") self.bin_new_path = os.path.join(self.mod_path, "bin_new") - self.manual_msg = "Download manually from: {r}.".format(r=self.repo_url) + self.manual_msg = f"Download manually from: {self.repo_url}." class _UpdateHandler: @@ -86,7 +86,7 @@ def _request_latest_json(url): @staticmethod def _check_path(file_path): if not os.path.exists(file_path): - raise PyXFormError("Expected path does not exist: {p}" "".format(p=file_path)) + raise PyXFormError(f"Expected path does not exist: {file_path}" "") else: return True @@ -96,7 +96,7 @@ def _read_json(file_path): Read the JSON file to a string. """ _UpdateHandler._check_path(file_path=file_path) - with open(file_path, mode="r") as in_file: + with open(file_path) as in_file: return json.load(in_file) @staticmethod @@ -114,7 +114,7 @@ def _read_last_check(file_path): Read the .last_check file. """ _UpdateHandler._check_path(file_path=file_path) - with open(file_path, mode="r") as in_file: + with open(file_path) as in_file: first_line = in_file.readline() try: last_check = datetime.strptime(first_line, UTC_FMT) @@ -192,7 +192,7 @@ def list(update_info): latest = _UpdateHandler._get_latest(update_info=update_info) latest_files = latest["assets"] if len(latest_files) == 0: - file_message = "- None!\n\n{m}".format(m=update_info.manual_msg) + file_message = f"- None!\n\n{update_info.manual_msg}" else: file_names = ["- {n}".format(n=x["name"]) for x in latest_files] file_message = "\n".join(file_names) @@ -229,8 +229,8 @@ def _find_download_url(update_info, json_data, file_name): urls_len = len(file_urls) if 0 == urls_len: raise PyXFormError( - "No files with the name '{n}' attached to release '{r}'." - "\n\n{h}".format(n=file_name, r=rel_name, h=update_info.manual_msg) + f"No files with the name '{file_name}' attached to release '{rel_name}'." + f"\n\n{update_info.manual_msg}" ) elif 1 < urls_len: raise PyXFormError( @@ -558,8 +558,8 @@ def _install_check(bin_file_path=None): def _build_validator_menu(main_subparser, validator_name, updater_instance): main = main_subparser.add_parser( validator_name.lower(), - description="{v} Sub-menu".format(v=validator_name), - help="For help, use '{v} -h'.".format(v=validator_name.lower()), + description=f"{validator_name} Sub-menu", + help=f"For help, use '{validator_name.lower()} -h'.", ) subs = main.add_subparsers(metavar="") diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index b38d494b1..ce09682c4 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -113,20 +113,18 @@ def request_get(url): with closing(urlopen(r)) as u: content = u.read() if len(content) == 0: - raise PyXFormError("Empty response from URL: '{u}'.".format(u=url)) + raise PyXFormError(f"Empty response from URL: '{url}'.") else: return content except HTTPError as http_err: raise PyXFormError( - "Unable to fulfill request. Error code: '{c}'. " - "Reason: '{r}'. URL: '{u}'." - "".format(r=http_err.reason, c=http_err.code, u=url) + f"Unable to fulfill request. Error code: '{http_err.code}'. " + f"Reason: '{http_err.reason}'. URL: '{url}'." + "" ) from http_err except URLError as url_err: raise PyXFormError( - "Unable to reach a server. Reason: {r}. " "URL: {u}".format( - r=url_err.reason, u=url - ) + f"Unable to reach a server. Reason: {url_err.reason}. " f"URL: {url}" ) from url_err @@ -195,7 +193,7 @@ def check_readable(file_path, retry_limit=10, wait_seconds=0.5): def catch_try(): try: - with open(file_path, mode="r"): + with open(file_path): return True except OSError: return False @@ -206,5 +204,5 @@ def catch_try(): tries += 1 time.sleep(wait_seconds) else: - raise OSError("Could not read file: {f}".format(f=file_path)) + raise OSError(f"Could not read file: {file_path}") return True diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index f93e50462..178239ce3 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -396,7 +396,7 @@ def _get_question_from_object(self, obj, type=None): ref = obj["nodeset"] except KeyError as node_err: raise PyXFormError( - 'Cannot find "ref" or "nodeset" in {}'.format(repr(obj)) + f'Cannot find "ref" or "nodeset" in {obj!r}' ) from node_err question = { "ref": ref, diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 74c2a1288..7a165a569 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -424,7 +424,7 @@ def workbook_to_json( workbook_dict = {x.lower(): y for x, y in workbook_dict.items()} workbook_keys = workbook_dict.keys() if constants.SURVEY not in workbook_dict: - msg = "You must have a sheet named '{k}'. ".format(k=constants.SURVEY) + msg = f"You must have a sheet named '{constants.SURVEY}'. " similar = find_sheet_misspellings(key=constants.SURVEY, keys=workbook_keys) if similar is not None: msg += similar @@ -609,10 +609,7 @@ def workbook_to_json( "allow_choice_duplicates setting to 'yes'. Learn more: https://xlsform.org/#choice-names.".format( list_name, ", ".join( - [ - "'{}'".format(dupe) - for dupe in choice_duplicates - ] + [f"'{dupe}'" for dupe in choice_duplicates] ), ) ) @@ -1592,7 +1589,7 @@ def get_filename(path): """ Get the extensionless filename from a path """ - return os.path.splitext((os.path.basename(path)))[0] + return os.path.splitext(os.path.basename(path))[0] def parse_file_to_json( diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 4a1d01b03..8e09ff96a 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -327,7 +327,7 @@ def replace_prefix(d, prefix): def csv_to_dict(path_or_file): if isinstance(path_or_file, str): - csv_data = open(path_or_file, "r", encoding="utf-8", newline="") + csv_data = open(path_or_file, encoding="utf-8", newline="") else: csv_data = path_or_file diff --git a/pyxform/xlsparseutils.py b/pyxform/xlsparseutils.py index 58e773622..456c38d05 100644 --- a/pyxform/xlsparseutils.py +++ b/pyxform/xlsparseutils.py @@ -32,7 +32,7 @@ def find_sheet_misspellings(key: str, keys: "KeysView") -> "Optional[str]": msg = ( "When looking for a sheet named '{k}', the following sheets with " "similar names were found: {c}." - ).format(k=key, c=str(", ".join(("'{}'".format(c) for c in candidates)))) + ).format(k=key, c=str(", ".join(f"'{c}'" for c in candidates))) return msg else: return None diff --git a/tests/test_dynamic_default.py b/tests/test_dynamic_default.py index 0ea4ad4f5..79644f5df 100644 --- a/tests/test_dynamic_default.py +++ b/tests/test_dynamic_default.py @@ -802,7 +802,7 @@ def test_dynamic_default_performance__time(self): | | text | q{i} | Q{i} | if(../t2 = 'test', 1, 2) + 15 - int(1.2) | """ for count in (500, 1000, 2000): - questions = "\n".join((question.format(i=i) for i in range(1, count))) + questions = "\n".join(question.format(i=i) for i in range(1, count)) md = "".join((survey_header, questions)) def run(name, case): @@ -836,7 +836,7 @@ def test_dynamic_default_performance__memory(self): question = """ | | text | q{i} | Q{i} | if(../t2 = 'test', 1, 2) + 15 - int(1.2) | """ - questions = "\n".join((question.format(i=i) for i in range(1, 2000))) + questions = "\n".join(question.format(i=i) for i in range(1, 2000)) md = "".join((survey_header, questions)) process = psutil.Process(os.getpid()) pre_mem = process.memory_info().rss diff --git a/tests/test_external_instances_for_selects.py b/tests/test_external_instances_for_selects.py index 73e67d3e8..b42916757 100644 --- a/tests/test_external_instances_for_selects.py +++ b/tests/test_external_instances_for_selects.py @@ -441,7 +441,7 @@ def test_itemset_csv_generated_from_external_choices(self): self.assertIn(log_msg, [r.message for r in log.records]) self.assertTrue(os.path.exists(itemsets_path)) - with open(itemsets_path, "r") as csv: + with open(itemsets_path) as csv: rows = csv.readlines() # Should have the non-empty headers in the first row. self.assertEqual('"list_name","name","state","city"\n', rows[0]) diff --git a/tests/test_survey.py b/tests/test_survey.py index af4fb2374..cd2bec186 100644 --- a/tests/test_survey.py +++ b/tests/test_survey.py @@ -29,8 +29,8 @@ def test_many_xpath_references_do_not_hit_64_recursion_limit__many_to_one(self): {q} | | note | n | {n} | """.format( - q="\n".join((tmpl_q.format(i) for i in range(1, 250))), - n=" ".join((tmpl_n.format(i) for i in range(1, 250))), + q="\n".join(tmpl_q.format(i) for i in range(1, 250)), + n=" ".join(tmpl_n.format(i) for i in range(1, 250)), ), ) @@ -45,7 +45,7 @@ def test_many_xpath_references_do_not_hit_64_recursion_limit__many_to_many(self) {q} {n} """.format( - q="\n".join((tmpl_q.format(i) for i in range(1, 250))), - n="\n".join((tmpl_n.format(i) for i in range(1, 250))), + q="\n".join(tmpl_q.format(i) for i in range(1, 250)), + n="\n".join(tmpl_n.format(i) for i in range(1, 250)), ), ) diff --git a/tests/test_translations.py b/tests/test_translations.py index bc460bea6..7b3f2a561 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -396,8 +396,8 @@ def test_missing_translations_check_performance(self): | | c{i} | nb | lb-d | lb-e | """ for count in (500, 1000, 2000): - questions = "\n".join((question.format(i=i) for i in range(1, count))) - choice_lists = "\n".join((choice_list.format(i=i) for i in range(1, count))) + questions = "\n".join(question.format(i=i) for i in range(1, count)) + choice_lists = "\n".join(choice_list.format(i=i) for i in range(1, count)) md = "".join((survey_header, questions, choices_header, choice_lists)) def run(name, case): diff --git a/tests/test_xls2json.py b/tests/test_xls2json.py index b6a1370da..3d518b50b 100644 --- a/tests/test_xls2json.py +++ b/tests/test_xls2json.py @@ -101,7 +101,7 @@ def test_workbook_to_json__ignore_prefixed_name__choices(self): md=CHOICES.format(name=n), errored=True, error__contains=[self.err_choices_required], - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__ignore_prefixed_name__external_choices(self): @@ -113,7 +113,7 @@ def test_workbook_to_json__ignore_prefixed_name__external_choices(self): md=EXTERNAL_CHOICES.format(name=n), errored=True, error__contains=[self.err_ext_choices_required], - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__ignore_prefixed_name__settings(self): @@ -135,7 +135,7 @@ def test_workbook_to_json__ignore_prefixed_name__survey(self): md=SURVEY.format(name=n), errored=True, error__contains=[self.err_survey_required], - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_found__choices(self): @@ -149,7 +149,7 @@ def test_workbook_to_json__misspelled_found__choices(self): error__contains=[ self.err_choices_required, self.err_similar_found, - "'{}'".format(n), + f"'{n}'", ], ) @@ -206,7 +206,7 @@ def test_workbook_to_json__misspelled_found__external_choices(self): error__contains=[ self.err_ext_choices_required, self.err_similar_found, - "'{}'".format(n), + f"'{n}'", ], ) @@ -265,7 +265,7 @@ def test_workbook_to_json__misspelled_found__settings(self): self.assertPyxformXform( name="test", md=SETTINGS.format(name=n), - warnings__contains=[self.err_similar_found, "'{}'".format(n)], + warnings__contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_found__settings_exists(self): @@ -315,7 +315,7 @@ def test_workbook_to_json__misspelled_found__survey(self): error__contains=[ self.err_survey_required, self.err_similar_found, - "'{}'".format(n), + f"'{n}'", ], ) @@ -363,7 +363,7 @@ def test_workbook_to_json__misspelled_not_found__choices(self): name="test", md=CHOICES.format(name=n), errored=True, - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_not_found__external_choices(self): @@ -374,7 +374,7 @@ def test_workbook_to_json__misspelled_not_found__external_choices(self): name="test", md=EXTERNAL_CHOICES.format(name=n), errored=True, - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_not_found__settings(self): diff --git a/tests/xform_test_case/xlsform_spec_test.py b/tests/xform_test_case/xlsform_spec_test.py index b21ff08c1..eadabef88 100644 --- a/tests/xform_test_case/xlsform_spec_test.py +++ b/tests/xform_test_case/xlsform_spec_test.py @@ -35,8 +35,8 @@ def compare_xform(self, file_name: str, set_default_name: bool = True): survey = builder.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(self.output_path, warnings=warnings) # Compare with the expected output: - with open(expected_output_path, "r", encoding="utf-8") as ef, open( - self.output_path, "r", encoding="utf-8" + with open(expected_output_path, encoding="utf-8") as ef, open( + self.output_path, encoding="utf-8" ) as af: expected = ef.read() observed = af.read() From 7559f0547bfede2937f9e8a99b52db784ff985ea Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 04:20:15 +1100 Subject: [PATCH 13/22] dev: fix linter warnings from pyupgrade rules - UP031 printf-string-formatting replace percent-formats with f-strings --- pyproject.toml | 19 +------------------ pyxform/instance.py | 4 ++-- pyxform/question.py | 5 +++-- pyxform/section.py | 5 ++--- pyxform/survey.py | 10 ++++------ pyxform/util/enum.py | 8 ++++---- pyxform/utils.py | 2 +- pyxform/xform2json.py | 2 +- pyxform/xlsparseutils.py | 2 +- tests/utils.py | 2 +- 10 files changed, 20 insertions(+), 39 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 17733dffa..0cb92706e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,10 +51,6 @@ src = ["pyxform", "tests"] [tool.ruff.lint] # By default, ruff enables flake8's F rules, along with a subset of the E rules. -extend-select = [ - "I", # isort -] -# Potentially useful rule sets to enable in future: select = [ "B", # flake8-bugbear "C4", # flake8-comprehensions @@ -72,20 +68,7 @@ select = [ "S", # flake8-bandit # "SIM", # flake8-simplify "TRY", # tryceratops -# "UP", # pyupgrade - "UP004", - "UP005", - "UP008", - "UP009", - "UP015", - "UP018", - "UP020", - "UP024", - "UP028", - "UP030", - "UP032", - "UP034", - "UP035", + "UP", # pyupgrade "W", # pycodestyle warning ] ignore = [ diff --git a/pyxform/instance.py b/pyxform/instance.py index 775458425..eff6a1cfe 100644 --- a/pyxform/instance.py +++ b/pyxform/instance.py @@ -57,11 +57,11 @@ def to_xml(self): A horrible way to do this, but it works (until we need the attributes pumped out in order, etc) """ - open_str = """<%s id="%s">""" % (self._name, self._id) + open_str = f"""<{self._name} id="{self._id}">""" close_str = """""" % self._name vals = "" for k, v in self._answers.items(): - vals += "<%s>%s" % (k, str(v), k) + vals += f"<{k}>{v!s}" output = open_str + vals + close_str return output diff --git a/pyxform/question.py b/pyxform/question.py index 2b2a5fb70..684b40976 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -53,8 +53,9 @@ def xml_control(self): if nested_setvalues: for setvalue in nested_setvalues: msg = ( - "The question ${%s} is not user-visible so it can't be used as a calculation trigger for question ${%s}." - % (self.name, setvalue[0]) + f"The question ${{{self.name}}} is not user-visible " + "so it can't be used as a calculation trigger for " + f"question ${{{setvalue[0]}}}." ) raise PyXFormError(msg) return None diff --git a/pyxform/section.py b/pyxform/section.py index 54d6c0c85..30f467cce 100644 --- a/pyxform/section.py +++ b/pyxform/section.py @@ -22,9 +22,8 @@ def _validate_uniqueness_of_element_names(self): elem_lower = element.name.lower() if elem_lower in element_slugs: raise PyXFormError( - "There are more than one survey elements named '%s' " - "(case-insensitive) in the section named '%s'." - % (elem_lower, self.name) + f"There are more than one survey elements named '{elem_lower}' " + f"(case-insensitive) in the section named '{self.name}'." ) element_slugs.add(elem_lower) diff --git a/pyxform/survey.py b/pyxform/survey.py index 915676eba..167067552 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -450,10 +450,8 @@ def _generate_from_file_instances(element) -> Optional[InstanceInfo]: itemset = element.get("itemset") file_id, ext = os.path.splitext(itemset) if itemset and ext in EXTERNAL_INSTANCE_EXTENSIONS: - uri = "jr://%s/%s" % ( - "file" if ext in {".xml", ".geojson"} else "file-%s" % ext[1:], - itemset, - ) + file_ext = "file" if ext in {".xml", ".geojson"} else f"file-{ext[1:]}" + uri = f"jr://{file_ext}/{itemset}" return InstanceInfo( type="file", context="[type: {t}, name: {n}]".format( @@ -1064,8 +1062,8 @@ def _is_return_relative_path() -> bool: return False intro = ( - "There has been a problem trying to replace %s with the " - "XPath to the survey element named '%s'." % (matchobj.group(0), name) + f"There has been a problem trying to replace {matchobj.group(0)} with the " + f"XPath to the survey element named '{name}'." ) if name not in self._xpath: raise PyXFormError(intro + " There is no survey element with this name.") diff --git a/pyxform/util/enum.py b/pyxform/util/enum.py index 4870013a4..86d62976c 100644 --- a/pyxform/util/enum.py +++ b/pyxform/util/enum.py @@ -9,19 +9,19 @@ class StrEnum(str, Enum): def __new__(cls, *values): "values must already be of type `str`" if len(values) > 3: - raise TypeError("too many arguments for str(): %r" % (values,)) + raise TypeError(f"too many arguments for str(): {values!r}") if len(values) == 1: # it must be a string if not isinstance(values[0], str): - raise TypeError("%r is not a string" % (values[0],)) + raise TypeError(f"{values[0]!r} is not a string") if len(values) >= 2: # check that encoding argument is a string if not isinstance(values[1], str): - raise TypeError("encoding must be a string, not %r" % (values[1],)) + raise TypeError(f"encoding must be a string, not {values[1]!r}") if len(values) == 3: # check that errors argument is a string if not isinstance(values[2], str): - raise TypeError("errors must be a string, not %r" % (values[2])) + raise TypeError(f"errors must be a string, not {values[2]!r}") value = str(*values) member = str.__new__(cls, value) member._value_ = value diff --git a/pyxform/utils.py b/pyxform/utils.py index cccc93204..de1427bd4 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -93,7 +93,7 @@ def writexml(self, writer, indent="", addindent="", newl=""): class PatchedText(Text): def writexml(self, writer, indent="", addindent="", newl=""): """Same as original but no replacing double quotes with '"'.""" - data = "%s%s%s" % (indent, self.data, newl) + data = "".join((indent, self.data, newl)) if data: data = data.replace("&", "&").replace("<", "<").replace(">", ">") writer.write(data) diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 178239ce3..1279e7bcb 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -366,7 +366,7 @@ def _get_item_func(self, ref, name, item): rs = {} name_splits = name.split("/") rs["name"] = name_splits[0] - ref = "%s/%s" % (ref, rs["name"]) + ref = f"""{ref}/{rs["name"]}""" rs["ref"] = ref if name_splits.__len__() > 1: rs["type"] = "group" diff --git a/pyxform/xlsparseutils.py b/pyxform/xlsparseutils.py index 456c38d05..7b1c995de 100644 --- a/pyxform/xlsparseutils.py +++ b/pyxform/xlsparseutils.py @@ -7,7 +7,7 @@ # http://www.w3.org/TR/REC-xml/ TAG_START_CHAR = r"[a-zA-Z:_]" TAG_CHAR = r"[a-zA-Z:_0-9\-.]" -XFORM_TAG_REGEXP = "%(start)s%(char)s*" % {"start": TAG_START_CHAR, "char": TAG_CHAR} +XFORM_TAG_REGEXP = f"{TAG_START_CHAR}{TAG_CHAR}*" def find_sheet_misspellings(key: str, keys: "KeysView") -> "Optional[str]": diff --git a/tests/utils.py b/tests/utils.py index e68991fa2..aec95fe86 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -28,7 +28,7 @@ def build_survey(filename): def create_survey_from_fixture(fixture_name, filetype="xls", include_directory=False): - fixture_path = path_to_text_fixture("%s.%s" % (fixture_name, filetype)) + fixture_path = path_to_text_fixture(f"{fixture_name}.{filetype}") noop, section_dict = file_utils.load_file_to_dict(fixture_path) pkg = {"main_section": section_dict} if include_directory: From b2cb5eec85cde63a3cddab669e76bc10fec1d0ec Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 05:52:13 +1100 Subject: [PATCH 14/22] dev: rename test modules/methods for unittest runner instead of nose - nose 1.3.7 is very old and not using any advanced test features that would require a 3rd party test library like nose2 or pytest - default pattern is test_* for runner to pick up TestCases --- README.rst | 6 +- pyproject.toml | 7 +- pyxform/survey.py | 2 +- tests/{builder_tests.py => test_builder.py} | 0 ...nd_load_tests.py => test_dump_and_load.py} | 0 ...{file_utils_test.py => test_file_utils.py} | 0 tests/{group_test.py => test_group.py} | 0 ..._test_creation.py => test_j2x_creation.py} | 0 ...antiation.py => test_j2x_instantiation.py} | 0 ...question_tests.py => test_j2x_question.py} | 0 ...py => test_j2x_xform_build_preparation.py} | 0 ..._json.py => test_js2x_import_from_json.py} | 0 ...{json2xform_test.py => test_json2xform.py} | 0 tests/{loop_tests.py => test_loop.py} | 0 ...{settings_test.py => test_settings_xls.py} | 0 ...{tutorial_test.py => test_tutorial_xls.py} | 0 tests/test_xform2json.py | 99 ++++++++++++++++- ...xls2json_tests.py => test_xls2json_xls.py} | 0 .../{xls2xform_tests.py => test_xls2xform.py} | 0 tests/xform2json_test.py | 102 ------------------ ...lumnstest.py => test_attribute_columns.py} | 7 +- .../{bug_tests.py => test_bugs.py} | 18 ++-- ...form_spec_test.py => test_xlsform_spec.py} | 2 +- .../{xml_tests.py => test_xml.py} | 0 24 files changed, 113 insertions(+), 130 deletions(-) rename tests/{builder_tests.py => test_builder.py} (100%) rename tests/{dump_and_load_tests.py => test_dump_and_load.py} (100%) rename tests/{file_utils_test.py => test_file_utils.py} (100%) rename tests/{group_test.py => test_group.py} (100%) rename tests/{j2x_test_creation.py => test_j2x_creation.py} (100%) rename tests/{j2x_test_instantiation.py => test_j2x_instantiation.py} (100%) rename tests/{j2x_question_tests.py => test_j2x_question.py} (100%) rename tests/{j2x_test_xform_build_preparation.py => test_j2x_xform_build_preparation.py} (100%) rename tests/{js2x_test_import_from_json.py => test_js2x_import_from_json.py} (100%) rename tests/{json2xform_test.py => test_json2xform.py} (100%) rename tests/{loop_tests.py => test_loop.py} (100%) rename tests/{settings_test.py => test_settings_xls.py} (100%) rename tests/{tutorial_test.py => test_tutorial_xls.py} (100%) rename tests/{xls2json_tests.py => test_xls2json_xls.py} (100%) rename tests/{xls2xform_tests.py => test_xls2xform.py} (100%) delete mode 100644 tests/xform2json_test.py rename tests/xform_test_case/{attributecolumnstest.py => test_attribute_columns.py} (93%) rename tests/xform_test_case/{bug_tests.py => test_bugs.py} (97%) rename tests/xform_test_case/{xlsform_spec_test.py => test_xlsform_spec.py} (98%) rename tests/xform_test_case/{xml_tests.py => test_xml.py} (100%) diff --git a/README.rst b/README.rst index d949dbb5c..bf988ab96 100644 --- a/README.rst +++ b/README.rst @@ -83,11 +83,7 @@ To set up for development / contributing, first complete the above steps for "Ru You can run tests with:: - nosetests - -On Windows, use:: - - nosetests -v -v --traverse-namespace ./tests + python -m unittest Before committing, make sure to format and lint the code using ``ruff``:: diff --git a/pyproject.toml b/pyproject.toml index 0cb92706e..81c3ac671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,15 +8,14 @@ description = "A Python package to create XForms for ODK Collect." readme = "README.rst" requires-python = ">=3.7" dependencies = [ - "xlrd==2.0.1", - "openpyxl==3.1.2", - "defusedxml==0.7.1", + "xlrd==2.0.1", # Read XLS files + "openpyxl==3.1.2", # Read XLSX files + "defusedxml==0.7.1", # Parse XML ] [project.optional-dependencies] # Install with `pip install pyxform[dev]`. dev = [ - "nose==1.3.7", # Run tests "formencode==2.1.0", # Compare XML "lxml==5.1.0", # XPath test expressions "psutil==5.9.7", # Process info for performance tests diff --git a/pyxform/survey.py b/pyxform/survey.py index 167067552..bcd33d6b7 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -1175,7 +1175,7 @@ def print_xform_to_file( bad_languages = get_languages_with_bad_tags(translations) if bad_languages: warnings.append( - "\tThe following language declarations do not contain " + "The following language declarations do not contain " "valid machine-readable codes: " + ", ".join(bad_languages) + ". " diff --git a/tests/builder_tests.py b/tests/test_builder.py similarity index 100% rename from tests/builder_tests.py rename to tests/test_builder.py diff --git a/tests/dump_and_load_tests.py b/tests/test_dump_and_load.py similarity index 100% rename from tests/dump_and_load_tests.py rename to tests/test_dump_and_load.py diff --git a/tests/file_utils_test.py b/tests/test_file_utils.py similarity index 100% rename from tests/file_utils_test.py rename to tests/test_file_utils.py diff --git a/tests/group_test.py b/tests/test_group.py similarity index 100% rename from tests/group_test.py rename to tests/test_group.py diff --git a/tests/j2x_test_creation.py b/tests/test_j2x_creation.py similarity index 100% rename from tests/j2x_test_creation.py rename to tests/test_j2x_creation.py diff --git a/tests/j2x_test_instantiation.py b/tests/test_j2x_instantiation.py similarity index 100% rename from tests/j2x_test_instantiation.py rename to tests/test_j2x_instantiation.py diff --git a/tests/j2x_question_tests.py b/tests/test_j2x_question.py similarity index 100% rename from tests/j2x_question_tests.py rename to tests/test_j2x_question.py diff --git a/tests/j2x_test_xform_build_preparation.py b/tests/test_j2x_xform_build_preparation.py similarity index 100% rename from tests/j2x_test_xform_build_preparation.py rename to tests/test_j2x_xform_build_preparation.py diff --git a/tests/js2x_test_import_from_json.py b/tests/test_js2x_import_from_json.py similarity index 100% rename from tests/js2x_test_import_from_json.py rename to tests/test_js2x_import_from_json.py diff --git a/tests/json2xform_test.py b/tests/test_json2xform.py similarity index 100% rename from tests/json2xform_test.py rename to tests/test_json2xform.py diff --git a/tests/loop_tests.py b/tests/test_loop.py similarity index 100% rename from tests/loop_tests.py rename to tests/test_loop.py diff --git a/tests/settings_test.py b/tests/test_settings_xls.py similarity index 100% rename from tests/settings_test.py rename to tests/test_settings_xls.py diff --git a/tests/tutorial_test.py b/tests/test_tutorial_xls.py similarity index 100% rename from tests/tutorial_test.py rename to tests/test_tutorial_xls.py diff --git a/tests/test_xform2json.py b/tests/test_xform2json.py index 3980388c1..3a2e62ddc 100644 --- a/tests/test_xform2json.py +++ b/tests/test_xform2json.py @@ -1,11 +1,106 @@ """ -Test XForm2JSON functionality +Test xform2json module. """ import json +import os +from unittest import TestCase +from xml.etree.ElementTree import ParseError -from pyxform.builder import create_survey_element_from_dict +from pyxform.builder import create_survey_element_from_dict, create_survey_from_path +from pyxform.xform2json import _try_parse, create_survey_element_from_xml +from tests import test_output, utils from tests.pyxform_test_case import PyxformTestCase +from tests.xform_test_case.base import XFormTestCase + + +class DumpAndLoadXForm2JsonTests(XFormTestCase): + maxDiff = None + + def setUp(self): + self.excel_files = [ + "gps.xls", + # "include.xls", + "choice_filter_test.xlsx", + "specify_other.xls", + "loop.xls", + "text_and_integer.xls", + # todo: this is looking for json that is created (and + # deleted) by another test, is should just add that json + # to the directory. + # "include_json.xls", + "simple_loop.xls", + "yes_or_no_question.xls", + "xlsform_spec_test.xlsx", + "field-list.xlsx", + "table-list.xls", + "group.xls", + ] + self.surveys = {} + self.this_directory = os.path.dirname(__file__) + for filename in self.excel_files: + path = utils.path_to_text_fixture(filename) + self.surveys[filename] = create_survey_from_path(path) + + def test_load_from_dump(self): + for filename, survey in self.surveys.items(): + with self.subTest(msg=filename): + survey.json_dump() + survey_from_dump = create_survey_element_from_xml(survey.to_xml()) + expected = survey.to_xml() + observed = survey_from_dump.to_xml() + self.assertXFormEqual(expected, observed) + + def tearDown(self): + for survey in self.surveys.values(): + path = survey.name + ".json" + if os.path.exists(path): + os.remove(path) + + +class TestXMLParse(TestCase): + @classmethod + def setUpClass(cls): + cls.tidy_file = None + cls.xml = """\n1""" + + def tearDown(self): + if self.tidy_file is not None: + os.remove(self.tidy_file) + + def test_try_parse_with_string(self): + """Should return root node from XML string.""" + root = _try_parse(self.xml) + self.assertEqual("a", root.tag) + + def test_try_parse_with_path(self): + """Should return root node from XML file path.""" + xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") + self.tidy_file = xml_path + with open(xml_path, "w") as xml_file: + xml_file.write(self.xml) + root = _try_parse(xml_path) + self.assertEqual("a", root.tag) + + def test_try_parse_with_bad_path(self): + """Should raise IOError: file doesn't exist.""" + xml_path = os.path.join(test_output.PATH, "not_a_real_file.xyz") + with self.assertRaises(IOError): + _try_parse(xml_path) + + def test_try_parse_with_bad_string(self): + """Should raise IOError: string parse failed and its not a path.""" + with self.assertRaises(IOError): + _try_parse("not valid xml :(") + + def test_try_parse_with_bad_file(self): + """Should raise XMLSyntaxError: file exists but content is not valid.""" + xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") + self.tidy_file = xml_path + with open(xml_path, "w") as xml_file: + xml_file.write("not valid xml :(") + with self.assertRaises(ParseError): + _try_parse(xml_path) class TestXForm2JSON(PyxformTestCase): diff --git a/tests/xls2json_tests.py b/tests/test_xls2json_xls.py similarity index 100% rename from tests/xls2json_tests.py rename to tests/test_xls2json_xls.py diff --git a/tests/xls2xform_tests.py b/tests/test_xls2xform.py similarity index 100% rename from tests/xls2xform_tests.py rename to tests/test_xls2xform.py diff --git a/tests/xform2json_test.py b/tests/xform2json_test.py deleted file mode 100644 index f2e7c4ff7..000000000 --- a/tests/xform2json_test.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Test xform2json module. -""" -import os -from unittest import TestCase -from xml.etree.ElementTree import ParseError - -from pyxform.builder import create_survey_from_path -from pyxform.xform2json import _try_parse, create_survey_element_from_xml - -from tests import test_output, utils -from tests.pyxform_test_case import PyxformTestCase -from tests.xform_test_case.base import XFormTestCase - - -class DumpAndLoadXForm2JsonTests(XFormTestCase, PyxformTestCase): - maxDiff = None - - def setUp(self): - self.excel_files = [ - "gps.xls", - # "include.xls", - "choice_filter_test.xlsx", - "specify_other.xls", - "loop.xls", - "text_and_integer.xls", - # todo: this is looking for json that is created (and - # deleted) by another test, is should just add that json - # to the directory. - # "include_json.xls", - "simple_loop.xls", - "yes_or_no_question.xls", - "xlsform_spec_test.xlsx", - "field-list.xlsx", - "table-list.xls", - "group.xls", - ] - self.surveys = {} - self.this_directory = os.path.dirname(__file__) - for filename in self.excel_files: - path = utils.path_to_text_fixture(filename) - self.surveys[filename] = create_survey_from_path(path) - - def test_load_from_dump(self): - for filename, survey in self.surveys.items(): - with self.subTest(msg=filename): - survey.json_dump() - survey_from_dump = create_survey_element_from_xml(survey.to_xml()) - expected = survey.to_xml() - observed = survey_from_dump.to_xml() - self.assertXFormEqual(expected, observed) - - def tearDown(self): - for survey in self.surveys.values(): - path = survey.name + ".json" - if os.path.exists(path): - os.remove(path) - - -class TestXMLParse(TestCase): - @classmethod - def setUpClass(cls): - cls.tidy_file = None - cls.xml = """\n1""" - - def tearDown(self): - if self.tidy_file is not None: - os.remove(self.tidy_file) - - def test_try_parse_with_string(self): - """Should return root node from XML string.""" - root = _try_parse(self.xml) - self.assertEqual("a", root.tag) - - def test_try_parse_with_path(self): - """Should return root node from XML file path.""" - xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") - self.tidy_file = xml_path - with open(xml_path, "w") as xml_file: - xml_file.write(self.xml) - root = _try_parse(xml_path) - self.assertEqual("a", root.tag) - - def test_try_parse_with_bad_path(self): - """Should raise IOError: file doesn't exist.""" - xml_path = os.path.join(test_output.PATH, "not_a_real_file.xyz") - with self.assertRaises(IOError): - _try_parse(xml_path) - - def test_try_parse_with_bad_string(self): - """Should raise IOError: string parse failed and its not a path.""" - with self.assertRaises(IOError): - _try_parse("not valid xml :(") - - def test_try_parse_with_bad_file(self): - """Should raise XMLSyntaxError: file exists but content is not valid.""" - xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") - self.tidy_file = xml_path - with open(xml_path, "w") as xml_file: - xml_file.write("not valid xml :(") - with self.assertRaises(ParseError): - _try_parse(xml_path) diff --git a/tests/xform_test_case/attributecolumnstest.py b/tests/xform_test_case/test_attribute_columns.py similarity index 93% rename from tests/xform_test_case/attributecolumnstest.py rename to tests/xform_test_case/test_attribute_columns.py index 9f4500d3b..bafd4f600 100644 --- a/tests/xform_test_case/attributecolumnstest.py +++ b/tests/xform_test_case/test_attribute_columns.py @@ -3,7 +3,6 @@ """ import codecs import os -import unittest import pyxform @@ -14,7 +13,7 @@ class AttributeColumnsTest(XFormTestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "attribute_columns_test.xlsx" self.get_file_path(filename) expected_output_path = os.path.join( @@ -35,7 +34,3 @@ def runTest(self): with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: self.assertXFormEqual(expected_file.read(), actual_file.read()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/xform_test_case/bug_tests.py b/tests/xform_test_case/test_bugs.py similarity index 97% rename from tests/xform_test_case/bug_tests.py rename to tests/xform_test_case/test_bugs.py index b88c1d162..150bc33d5 100644 --- a/tests/xform_test_case/bug_tests.py +++ b/tests/xform_test_case/test_bugs.py @@ -19,7 +19,7 @@ class GroupNames(unittest.TestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "group_name_test.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -38,7 +38,7 @@ def runTest(self): class NotClosedGroup(unittest.TestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "not_closed_group_test.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -59,7 +59,7 @@ def runTest(self): class DuplicateColumns(unittest.TestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "duplicate_columns.xlsx" path_to_excel_file = os.path.join(example_xls.PATH, filename) # Get the xform output path: @@ -78,7 +78,7 @@ def runTest(self): class RepeatDateTest(XFormTestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "repeat_date_test.xls" self.get_file_path(filename) expected_output_path = os.path.join( @@ -102,7 +102,7 @@ def runTest(self): class XmlEscaping(XFormTestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "xml_escaping.xls" self.get_file_path(filename) expected_output_path = os.path.join( @@ -126,7 +126,7 @@ def runTest(self): class DefaultTimeTest(XFormTestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "default_time_demo.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -153,7 +153,7 @@ class ValidateWrapper(unittest.TestCase): maxDiff = None @staticmethod - def runTest(): + def test_conversion(self): filename = "ODKValidateWarnings.xlsx" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -169,7 +169,7 @@ def runTest(): class EmptyStringOnRelevantColumnTest(unittest.TestCase): - def runTest(self): + def test_conversion(self): filename = "ict_survey_fails.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) workbook_dict = pyxform.xls2json.parse_file_to_workbook_dict(path_to_excel_file) @@ -179,7 +179,7 @@ def runTest(self): class BadChoicesSheetHeaders(unittest.TestCase): - def runTest(self): + def test_conversion(self): filename = "spaces_in_choices_header.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) warnings = [] diff --git a/tests/xform_test_case/xlsform_spec_test.py b/tests/xform_test_case/test_xlsform_spec.py similarity index 98% rename from tests/xform_test_case/xlsform_spec_test.py rename to tests/xform_test_case/test_xlsform_spec.py index eadabef88..80c5ba5c2 100644 --- a/tests/xform_test_case/xlsform_spec_test.py +++ b/tests/xform_test_case/test_xlsform_spec.py @@ -66,7 +66,7 @@ class TestCalculateWithoutCalculation(TestCase): Just checks that calculate field without calculation raises a PyXFormError. """ - def runTest(self): + def test_conversion(self): filename = "calculate_without_calculation.xls" path_to_excel_file = os.path.join(example_xls.PATH, filename) self.assertRaises(PyXFormError, xls2json.parse_file_to_json, path_to_excel_file) diff --git a/tests/xform_test_case/xml_tests.py b/tests/xform_test_case/test_xml.py similarity index 100% rename from tests/xform_test_case/xml_tests.py rename to tests/xform_test_case/test_xml.py From 92d53a5fa1970383fb6bee6310c3140b14a0baef Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Tue, 16 Jan 2024 06:24:23 +1100 Subject: [PATCH 15/22] dev: update usages of codecs.open to use underlying built-in - remove redundant mode/encoding args - simplify unittest import in test_bugs.py --- pyxform/survey.py | 3 +- pyxform/utils.py | 5 +- pyxform/xform2json.py | 3 +- pyxform/xls2json.py | 6 +-- tests/pyxform_test_case.py | 3 +- tests/test_xls2json_xls.py | 19 +++----- .../xform_test_case/test_attribute_columns.py | 6 +-- tests/xform_test_case/test_bugs.py | 46 ++++++++----------- 8 files changed, 34 insertions(+), 57 deletions(-) diff --git a/pyxform/survey.py b/pyxform/survey.py index bcd33d6b7..22b237f29 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -1,7 +1,6 @@ """ Survey module with XForm Survey objects and utility functions. """ -import codecs import os import re import tempfile @@ -1155,7 +1154,7 @@ def print_xform_to_file( if not path: path = self._print_name + ".xml" try: - with codecs.open(path, mode="w", encoding="utf-8") as file_obj: + with open(path, mode="w", encoding="utf-8") as file_obj: if pretty_print: file_obj.write(self._to_pretty_xml()) else: diff --git a/pyxform/utils.py b/pyxform/utils.py index de1427bd4..bc555c780 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -1,7 +1,6 @@ """ pyxform utils module. """ -import codecs import copy import csv import json @@ -165,8 +164,8 @@ def get_pyobj_from_json(str_or_path): """ try: # see if treating str_or_path as a path works - fp = codecs.open(str_or_path, mode="r", encoding="utf-8") - doc = json.load(fp) + with open(str_or_path, encoding="utf-8") as fp: + doc = json.load(fp) except (JSONDecodeError, OSError): # if it doesn't work load the text doc = json.loads(str_or_path) diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 1279e7bcb..e6d458b83 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -1,7 +1,6 @@ """ xform2json module - Transform an XForm to a JSON dictionary. """ -import codecs import copy import json import logging @@ -717,6 +716,6 @@ def replace_function(match): def write_object_to_file(filename, obj): """Writes to a JSON filename the dictionary obj.""" - with codecs.open(filename, "w", encoding="utf-8") as output_file: + with open(filename, "w", encoding="utf-8") as output_file: output_file.write(json.dumps(obj, indent=2)) logger.info("object written to file: %s", filename) diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 7a165a569..0d1ae1068 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -1,7 +1,6 @@ """ A Python script to convert excel files into JSON. """ -import codecs import json import os import re @@ -38,9 +37,8 @@ def print_pyobj_to_json(pyobj, path=None): or stdout if no file is specified """ if path: - fp = codecs.open(path, mode="w", encoding="utf-8") - json.dump(pyobj, fp=fp, ensure_ascii=False, indent=4) - fp.close() + with open(path, mode="w", encoding="utf-8") as fp: + json.dump(pyobj, fp=fp, ensure_ascii=False, indent=4) else: sys.stdout.write(json.dumps(pyobj, ensure_ascii=False, indent=4)) diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 95b81588c..355c97e4f 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -1,7 +1,6 @@ """ PyxformTestCase base class using markdown to define the XLSForm. """ -import codecs import logging import os import re @@ -131,7 +130,7 @@ def _run_odk_validate(xml): tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) tmp.close() try: - with codecs.open(tmp.name, mode="w", encoding="utf-8") as fp: + with open(tmp.name, mode="w", encoding="utf-8") as fp: fp.write(xml) fp.close() check_xform(tmp.name) diff --git a/tests/test_xls2json_xls.py b/tests/test_xls2json_xls.py index 564ba4091..51b53da08 100644 --- a/tests/test_xls2json_xls.py +++ b/tests/test_xls2json_xls.py @@ -1,7 +1,6 @@ """ Testing simple cases for Xls2Json """ -import codecs import json import os from unittest import TestCase @@ -32,14 +31,11 @@ def test_simple_yes_or_no_question(self): ) x = SurveyReader(path_to_excel_file, default_name="yes_or_no_question") x_results = x.to_json_dict() - with codecs.open(output_path, mode="w", encoding="utf-8") as fp: + with open(output_path, mode="w", encoding="utf-8") as fp: json.dump(x_results, fp=fp, ensure_ascii=False, indent=4) # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: - expected_json = json.load(expected_file) - actual_json = json.load(actual_file) - self.assertEqual(expected_json, actual_json) + with open(expected_output_path) as expected, open(output_path) as observed: + self.assertEqual(json.load(expected), json.load(observed)) def test_hidden(self): x = SurveyReader(utils.path_to_text_fixture("hidden.xls"), default_name="hidden") @@ -127,14 +123,11 @@ def test_table(self): ) x = SurveyReader(path_to_excel_file, default_name="simple_loop") x_results = x.to_json_dict() - with codecs.open(output_path, mode="w", encoding="utf-8") as fp: + with open(output_path, mode="w", encoding="utf-8") as fp: json.dump(x_results, fp=fp, ensure_ascii=False, indent=4) # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: - expected_json = json.load(expected_file) - actual_json = json.load(actual_file) - self.assertEqual(expected_json, actual_json) + with open(expected_output_path) as expected, open(output_path) as observed: + self.assertEqual(json.load(expected), json.load(observed)) def test_choice_filter_choice_fields(self): """ diff --git a/tests/xform_test_case/test_attribute_columns.py b/tests/xform_test_case/test_attribute_columns.py index bafd4f600..f0e9bcc7f 100644 --- a/tests/xform_test_case/test_attribute_columns.py +++ b/tests/xform_test_case/test_attribute_columns.py @@ -1,7 +1,6 @@ """ Some tests for the new (v0.9) spec is properly implemented. """ -import codecs import os import pyxform @@ -31,6 +30,5 @@ def test_conversion(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) + with open(expected_output_path) as expected, open(self.output_path) as observed: + self.assertXFormEqual(expected.read(), observed.read()) diff --git a/tests/xform_test_case/test_bugs.py b/tests/xform_test_case/test_bugs.py index 150bc33d5..11b11218a 100644 --- a/tests/xform_test_case/test_bugs.py +++ b/tests/xform_test_case/test_bugs.py @@ -1,9 +1,8 @@ """ Some tests for the new (v0.9) spec is properly implemented. """ -import codecs import os -import unittest +from unittest import TestCase import pyxform from pyxform.errors import PyXFormError @@ -16,7 +15,7 @@ from tests.xform_test_case.base import XFormTestCase -class GroupNames(unittest.TestCase): +class GroupNames(TestCase): maxDiff = None def test_conversion(self): @@ -35,7 +34,7 @@ def test_conversion(self): survey.print_xform_to_file(output_path, warnings=warnings) -class NotClosedGroup(unittest.TestCase): +class NotClosedGroup(TestCase): maxDiff = None def test_conversion(self): @@ -56,7 +55,7 @@ def test_conversion(self): survey.print_xform_to_file(output_path, warnings=warnings) -class DuplicateColumns(unittest.TestCase): +class DuplicateColumns(TestCase): maxDiff = None def test_conversion(self): @@ -94,9 +93,8 @@ def test_conversion(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) + with open(expected_output_path) as expected, open(self.output_path) as observed: + self.assertXFormEqual(expected.read(), observed.read()) class XmlEscaping(XFormTestCase): @@ -118,9 +116,8 @@ def test_conversion(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) + with open(expected_output_path) as expected, open(self.output_path) as observed: + self.assertXFormEqual(expected.read(), observed.read()) class DefaultTimeTest(XFormTestCase): @@ -144,12 +141,11 @@ def test_conversion(self): survey.print_xform_to_file(output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) + with open(expected_output_path) as expected, open(output_path) as observed: + self.assertXFormEqual(expected.read(), observed.read()) -class ValidateWrapper(unittest.TestCase): +class ValidateWrapper(TestCase): maxDiff = None @staticmethod @@ -168,7 +164,7 @@ def test_conversion(self): survey.print_xform_to_file(output_path, warnings=warnings) -class EmptyStringOnRelevantColumnTest(unittest.TestCase): +class EmptyStringOnRelevantColumnTest(TestCase): def test_conversion(self): filename = "ict_survey_fails.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) @@ -178,7 +174,7 @@ def test_conversion(self): workbook_dict["survey"][0]["bind: relevant"].strip() -class BadChoicesSheetHeaders(unittest.TestCase): +class BadChoicesSheetHeaders(TestCase): def test_conversion(self): filename = "spaces_in_choices_header.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) @@ -209,7 +205,7 @@ def test_values_with_spaces_are_cleaned(self): ) -class TestChoiceNameAsType(unittest.TestCase): +class TestChoiceNameAsType(TestCase): def test_choice_name_as_type(self): filename = "choice_name_as_type.xls" path_to_excel_file = os.path.join(example_xls.PATH, filename) @@ -218,7 +214,7 @@ def test_choice_name_as_type(self): self.assertTrue(has_external_choices(survey_dict)) -class TestBlankSecondRow(unittest.TestCase): +class TestBlankSecondRow(TestCase): def test_blank_second_row(self): filename = "blank_second_row.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) @@ -227,7 +223,7 @@ def test_blank_second_row(self): self.assertTrue(len(survey_dict) > 0) -class TestXLDateAmbigous(unittest.TestCase): +class TestXLDateAmbigous(TestCase): """Test non standard sheet with exception is processed successfully.""" def test_xl_date_ambigous(self): @@ -239,7 +235,7 @@ def test_xl_date_ambigous(self): self.assertTrue(len(survey_dict) > 0) -class TestXLDateAmbigousNoException(unittest.TestCase): +class TestXLDateAmbigousNoException(TestCase): """Test date values that exceed the workbook datemode value. (This would cause an exception with xlrd, but openpyxl handles it).""" @@ -251,7 +247,7 @@ def test_xl_date_ambigous_no_exception(self): self.assertEqual(survey_dict["survey"][4]["default"], "1900-01-01 00:00:00") -class TestSpreadSheetFilesWithMacrosAreAllowed(unittest.TestCase): +class TestSpreadSheetFilesWithMacrosAreAllowed(TestCase): """Test that spreadsheets with .xlsm extension are allowed""" def test_xlsm_files_are_allowed(self): @@ -261,14 +257,10 @@ def test_xlsm_files_are_allowed(self): self.assertIsInstance(result, dict) -class TestBadCalculation(unittest.TestCase): +class TestBadCalculation(TestCase): """Bad calculation should not kill the application""" def test_bad_calculate_javarosa_error(self): filename = "bad_calc.xml" test_xml = os.path.join(test_output.PATH, filename) self.assertRaises(ODKValidateError, check_xform, test_xml) - - -if __name__ == "__main__": - unittest.main() From 5533905bc8b8785083225e83ab245484d51dda20 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 17 Jan 2024 02:44:43 +1100 Subject: [PATCH 16/22] dev: fix ResourceWarning during tests, broken test, string formatting - xls2json_backends.py: both XLS and XLSX open and read the respective files, do processing, then close the file. But if an error occurs during processing then the file is left open. A ResourceWarning is raised when python cleans up the file handle. The ResourceWarning is visible during tests because test libraries remove the normal suppression of warnings during test runs. So normally a user probably wouldn't see it. The fix is to make sure the workbook object and file are cleaned up and closed with a contextmanager + try-except block. - test_bugs.py: fixed test that had been broken for a while. It was not being executed by nose test runner for some reason. After converting tests to unittest, it ran and failed. After doing some git archaeology it seems that the 3rd warning that was expected was one about Google Sheets not supporting certain file names, but while that warning was later removed, the test was not updated. The 2nd is commented on. - fixed strange string concatentation on the same line e.g. "a" "b". --- pyxform/validators/updater.py | 12 +-- pyxform/xls2json.py | 4 +- pyxform/xls2json_backends.py | 148 ++++++++++++++++++----------- tests/xform_test_case/test_bugs.py | 5 +- 4 files changed, 100 insertions(+), 69 deletions(-) diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index 9b5fea5b0..732eaa391 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -171,7 +171,7 @@ def _get_latest(update_info): @staticmethod def _get_release_message(json_data): - template = "- Tag name = {tag_name}\n" "- Tag URL = {tag_url}\n\n" + template = "- Tag name = {tag_name}\n- Tag URL = {tag_url}\n\n" return template.format( tag_name=json_data["tag_name"], tag_url=json_data["html_url"] ) @@ -219,9 +219,7 @@ def _find_download_url(update_info, json_data, file_name): if len(files) == 0: raise PyXFormError( - "No files attached to release '{r}'.\n\n{h}" "".format( - r=rel_name, h=update_info.manual_msg - ) + f"No files attached to release '{rel_name}'.\n\n{update_info.manual_msg}" ) file_urls = [x["browser_download_url"] for x in files if x["name"] == file_name] @@ -269,7 +267,7 @@ def _get_bin_paths(update_info, file_path): main_bin = "*validate" else: raise PyXFormError( - "Did not find a supported main binary for file: {p}.\n\n{h}" "".format( + "Did not find a supported main binary for file: {p}.\n\n{h}".format( p=file_path, h=update_info.manual_msg ) ) @@ -309,9 +307,7 @@ def _unzip_find_jobs(open_zip_file, bin_paths, out_path): zip_jobs[file_out_path] = zip_item if len(bin_paths) != len(zip_jobs.keys()): raise PyXFormError( - "Expected {e} zip job files, found: {c}" "".format( - e=len(bin_paths), c=len(zip_jobs.keys()) - ) + f"Expected {len(bin_paths)} zip job files, found: {len(zip_jobs.keys())}" ) return zip_jobs diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 0d1ae1068..5701edba3 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -556,7 +556,7 @@ def workbook_to_json( if "name" not in option: info = "[list_name : " + list_name + "]" raise PyXFormError( - "On the choices sheet there is " "a option with no name. " + info + "On the choices sheet there is a option with no name. " + info ) if "label" not in option: info = "[list_name : " + list_name + "]" @@ -564,7 +564,7 @@ def workbook_to_json( "On the choices sheet there is a option with no label. " + info ) # chrislrobert's fix for a cryptic error message: - # see: https://code.google.com/p/opendatakit/issues/detail?id=832&start=200 + # see: https://code.google.com/p/opendatakit/issues/detail?id=833&start=200 option_keys = list(option.keys()) for headername in option_keys: # Using warnings and removing the bad columns diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 8e09ff96a..eca89014b 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -3,17 +3,23 @@ """ import csv import datetime +import os import re from collections import OrderedDict +from contextlib import closing from functools import reduce from io import StringIO from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union from zipfile import BadZipFile -import openpyxl import xlrd from openpyxl.cell import Cell as pyxlCell +from openpyxl.reader.excel import ExcelReader +from openpyxl.workbook import Workbook as pyxlWorkbook +from openpyxl.worksheet.worksheet import Worksheet as pyxlWorksheet +from xlrd.book import Book as xlrdBook from xlrd.sheet import Cell as xlrdCell +from xlrd.sheet import Sheet as xlrdSheet from xlrd.xldate import XLDateAmbiguous from pyxform import constants @@ -120,21 +126,16 @@ def xls_to_dict(path_or_file): equal to the cell value for that row and column. All the keys and leaf elements are unicode text. """ - try: - if isinstance(path_or_file, str): - workbook = xlrd.open_workbook(filename=path_or_file) - else: - workbook = xlrd.open_workbook(file_contents=path_or_file.read()) - except xlrd.XLRDError as read_err: - raise PyXFormError("Error reading .xls file: %s" % read_err) from read_err - def xls_clean_cell(cell: xlrdCell, row_n: int, col_key: str) -> Optional[str]: + def xls_clean_cell( + wb: xlrdBook, wb_sheet: xlrdSheet, cell: xlrdCell, row_n: int, col_key: str + ) -> Optional[str]: value = cell.value if isinstance(value, str): value = value.strip() if not is_empty(value): try: - return xls_value_to_unicode(value, cell.ctype, workbook.datemode) + return xls_value_to_unicode(value, cell.ctype, wb.datemode) except XLDateAmbiguous as date_err: raise PyXFormError( XL_DATE_AMBIGOUS_MSG % (wb_sheet.name, col_key, row_n) @@ -142,39 +143,59 @@ def xls_clean_cell(cell: xlrdCell, row_n: int, col_key: str) -> Optional[str]: return None - def xls_to_dict_normal_sheet(sheet): + def xls_to_dict_normal_sheet(wb: xlrdBook, wb_sheet: xlrdSheet): # XLS format: max cols 256, max rows 65536 - first_row = (c.value for c in next(sheet.get_rows(), [])) + first_row = (c.value for c in next(wb_sheet.get_rows(), [])) headers = get_excel_column_headers(first_row=first_row) row_iter = ( - tuple(sheet.cell(r, c) for c in range(len(headers))) - for r in range(1, sheet.nrows) + tuple(wb_sheet.cell(r, c) for c in range(len(headers))) + for r in range(1, wb_sheet.nrows) ) - rows = get_excel_rows(headers=headers, rows=row_iter, cell_func=xls_clean_cell) + + # Inject wb/sheet as closure since functools.partial isn't typing friendly. + def clean_func(cell: xlrdCell, row_n: int, col_key: str) -> Optional[str]: + return xls_clean_cell( + wb=wb, wb_sheet=wb_sheet, cell=cell, row_n=row_n, col_key=col_key + ) + + rows = get_excel_rows(headers=headers, rows=row_iter, cell_func=clean_func) column_header_list = [key for key in headers if key is not None] return rows, _list_to_dict_list(column_header_list) - result_book = OrderedDict() - for wb_sheet in workbook.sheets(): - # Note that the sheet exists but do no further processing here. - result_book[wb_sheet.name] = [] - # Do not process sheets that have nothing to do with XLSForm. - if wb_sheet.name not in constants.SUPPORTED_SHEET_NAMES: - if len(workbook.sheets()) == 1: - ( - result_book[constants.SURVEY], - result_book["%s_header" % constants.SURVEY], - ) = xls_to_dict_normal_sheet(wb_sheet) + def process_workbook(wb: xlrdBook): + result_book = OrderedDict() + for wb_sheet in wb.sheets(): + # Note that the sheet exists but do no further processing here. + result_book[wb_sheet.name] = [] + # Do not process sheets that have nothing to do with XLSForm. + if wb_sheet.name not in constants.SUPPORTED_SHEET_NAMES: + if len(wb.sheets()) == 1: + ( + result_book[constants.SURVEY], + result_book["%s_header" % constants.SURVEY], + ) = xls_to_dict_normal_sheet(wb=wb, wb_sheet=wb_sheet) + else: + continue else: - continue - else: - ( - result_book[wb_sheet.name], - result_book["%s_header" % wb_sheet.name], - ) = xls_to_dict_normal_sheet(wb_sheet) + ( + result_book[wb_sheet.name], + result_book["%s_header" % wb_sheet.name], + ) = xls_to_dict_normal_sheet(wb=wb, wb_sheet=wb_sheet) + return result_book - workbook.release_resources() - return result_book + try: + if isinstance(path_or_file, (str, bytes, os.PathLike)): + file = open(path_or_file, mode="rb") + else: + file = path_or_file + with closing(file) as wb_file: + workbook = xlrd.open_workbook(file_contents=wb_file.read()) + try: + return process_workbook(wb=workbook) + finally: + workbook.release_resources() + except xlrd.XLRDError as read_err: + raise PyXFormError("Error reading .xls file: %s" % read_err) from read_err def xls_value_to_unicode(value, value_type, datemode) -> str: @@ -216,10 +237,6 @@ def xlsx_to_dict(path_or_file): equal to the cell value for that row and column. All the keys and leaf elements are strings. """ - try: - workbook = openpyxl.open(filename=path_or_file, read_only=True, data_only=True) - except (OSError, BadZipFile, KeyError) as read_err: - raise PyXFormError("Error reading .xlsx file: %s" % read_err) from read_err def xlsx_clean_cell(cell: pyxlCell, row_n: int, col_key: str) -> Optional[str]: value = cell.value @@ -230,7 +247,7 @@ def xlsx_clean_cell(cell: pyxlCell, row_n: int, col_key: str) -> Optional[str]: return None - def xlsx_to_dict_normal_sheet(sheet): + def xlsx_to_dict_normal_sheet(sheet: pyxlWorksheet): # XLSX format: max cols 16384, max rows 1048576 first_row = (c.value for c in next(sheet.rows, [])) headers = get_excel_column_headers(first_row=first_row) @@ -239,28 +256,43 @@ def xlsx_to_dict_normal_sheet(sheet): column_header_list = [key for key in headers if key is not None] return rows, _list_to_dict_list(column_header_list) - result_book = OrderedDict() - for sheetname in workbook.sheetnames: - wb_sheet = workbook[sheetname] - # Note that the sheet exists but do no further processing here. - result_book[sheetname] = [] - # Do not process sheets that have nothing to do with XLSForm. - if sheetname not in constants.SUPPORTED_SHEET_NAMES: - if len(workbook.sheetnames) == 1: + def process_workbook(wb: pyxlWorkbook): + result_book = OrderedDict() + for sheetname in wb.sheetnames: + wb_sheet = wb[sheetname] + # Note that the sheet exists but do no further processing here. + result_book[sheetname] = [] + # Do not process sheets that have nothing to do with XLSForm. + if sheetname not in constants.SUPPORTED_SHEET_NAMES: + if len(wb.sheetnames) == 1: + ( + result_book[constants.SURVEY], + result_book[f"{constants.SURVEY}_header"], + ) = xlsx_to_dict_normal_sheet(wb_sheet) + else: + continue + else: ( - result_book[constants.SURVEY], - result_book[f"{constants.SURVEY}_header"], + result_book[sheetname], + result_book[f"{sheetname}_header"], ) = xlsx_to_dict_normal_sheet(wb_sheet) - else: - continue - else: - ( - result_book[sheetname], - result_book[f"{sheetname}_header"], - ) = xlsx_to_dict_normal_sheet(wb_sheet) + return result_book - workbook.close() - return result_book + try: + if isinstance(path_or_file, (str, bytes, os.PathLike)): + file = open(path_or_file, mode="rb") + else: + file = path_or_file + with closing(file) as wb_file: + reader = ExcelReader(wb_file, read_only=True, data_only=True) + reader.read() + try: + return process_workbook(wb=reader.wb) + finally: + reader.wb.close() + reader.archive.close() + except (OSError, BadZipFile, KeyError) as read_err: + raise PyXFormError("Error reading .xlsx file: %s" % read_err) from read_err def xlsx_value_to_str(value) -> str: diff --git a/tests/xform_test_case/test_bugs.py b/tests/xform_test_case/test_bugs.py index 11b11218a..31091c465 100644 --- a/tests/xform_test_case/test_bugs.py +++ b/tests/xform_test_case/test_bugs.py @@ -184,7 +184,10 @@ def test_conversion(self): default_name="spaces_in_choices_header", warnings=warnings, ) - self.assertEqual(len(warnings), 3, "Found " + str(len(warnings)) + " warnings") + # The "column with no header" warning is probably not reachable since XLS/X + # pre-processing ignores any columns without a header. + observed = [w for w in warnings if "Headers cannot include spaces" in w] + self.assertEqual(1, len(observed), warnings) def test_values_with_spaces_are_cleaned(self): """ From 0d5af8faebd39f6098fe39cec694c3845fd79ada Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 17 Jan 2024 19:08:44 +1100 Subject: [PATCH 17/22] dev: fix expected output in attribute_columns_test - nose tests were not running this test for a while. The changes are: - secondary instances / itemsets replacing inline choices. - for media itext with no translation, the value/form not output - probable bug: custom attributes not applied to repeat template (they appear in first repeat but not the jr:template). --- .../attribute_columns_test.xml | 384 +++++------------- 1 file changed, 112 insertions(+), 272 deletions(-) diff --git a/tests/test_expected_output/attribute_columns_test.xml b/tests/test_expected_output/attribute_columns_test.xml index 89071bd05..61c895e3d 100644 --- a/tests/test_expected_output/attribute_columns_test.xml +++ b/tests/test_expected_output/attribute_columns_test.xml @@ -3,15 +3,24 @@ Spec test - + - - - + + Yes + + + No - + jr://images/a.jpg + + jr://images/b.jpg + + + - + - @@ -36,9 +45,6 @@ - - - Yes - - @@ -48,9 +54,6 @@ - - - Yes - - @@ -66,12 +69,6 @@ - - - No - - - jr://images/a.jpg - - @@ -93,21 +90,9 @@ - - - jr://images/b.jpg - - - Yes - - - No - - - - Yes - - @@ -132,9 +117,6 @@ - - - No - - @@ -144,9 +126,6 @@ - - - No - - @@ -156,45 +135,18 @@ - - - No - - - - Yes - - - Yes - - - jr://images/b.jpg - - - Yes - - - No - - a note - - Yes - - - No - jr://images/img_test.jpg - - No - - @@ -209,6 +161,14 @@ + + 是 + + + 没有 + + + - @@ -233,15 +193,9 @@ - - - jr://images/- - ni hao - - 是 - - @@ -251,16 +205,11 @@ - - - 是 - - - jr://images/- jr://audio/chinese_audio.wav - jr://video/- 您好 @@ -269,12 +218,6 @@ - - - 没有 - - - jr://images/- - - @@ -296,30 +239,13 @@ - - - jr://images/- - - - 是 - - - 没有 - - - 没有 - - - 是 - - - - - jr://images/- - + - @@ -335,9 +261,6 @@ - - - 没有 - - @@ -347,9 +270,6 @@ - - - 没有 - - @@ -365,39 +285,15 @@ - - - 是 - - - 是 - - - jr://images/- - - - 是 - - - 没有 - - - - - 是 - - - 没有 - - - - 没有 - - @@ -412,6 +308,14 @@ + + - + + + - + + + video_test @@ -427,9 +331,6 @@ constrained decimal - - - - This launches a fictional application to get an integer result. @@ -439,9 +340,6 @@ select multiple test - - - - deviceid_test_output: @@ -452,16 +350,11 @@ start test output: - - - - label-test jr://images/img_test_2.jpg - jr://audio/- - jr://video/- text_image_audio_video_test @@ -469,12 +362,6 @@ date_test - - - - - - jr://images/- - autocomplete_test @@ -483,44 +370,24 @@ - - - - - image_test datetime_test - - jr://images/- - today_test_output: - - - - audio_test - - jr://images/- - - - - - - - - - a integer - - jr://images/- - + labeled select group test @@ -533,17 +400,11 @@ Skip to end - - - - Your name is time_test - - - - simserial_test_output: @@ -563,33 +424,12 @@ - - - - - - - - - - - jr://images/- - - - - - - - - - You entered an email address - - - - - - - - - required_text @@ -623,7 +463,23 @@ - + + + + + + + + + + + + + + + + + @@ -675,6 +531,30 @@ + + + + yes_no-0 + yes + + + yes_no-1 + no + + + + + + + a_b-0 + a + + + a_b-1 + b + + + @@ -734,14 +614,10 @@ @@ -771,25 +647,17 @@ - - - - + + + From 3ffc8c7e5e208a1c446c9b95e516b436790f8629 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 17 Jan 2024 20:00:17 +1100 Subject: [PATCH 18/22] dev: update github actions - bump actions version numbers - update syntax for install, build, test --- .github/workflows/release.yml | 6 +++--- .github/workflows/verify.yml | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26f7d5437..a3f9e939b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,14 +12,14 @@ jobs: python: ['3.8'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} # Install dependencies. - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Python cache with dependencies. id: python-cache with: diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index d4c3e44cb..f8648b107 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -10,14 +10,14 @@ jobs: python: ['3.8'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} # Install dependencies. - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Python cache with dependencies. id: python-cache with: @@ -26,7 +26,7 @@ jobs: - name: Install dependencies. run: | python -m pip install --upgrade pip - pip install -r dev_requirements.pip + pip install -e .[dev] pip list # Linter. @@ -45,38 +45,38 @@ jobs: - os: windows-latest windows_nose_args: --traverse-namespace ./tests steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} # Install dependencies. - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Python cache with dependencies. id: python-cache with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev_requirements.pip') }} + key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }} - name: Install dependencies. run: | python -m pip install --upgrade pip - pip install -r dev_requirements.pip + pip install -e .[dev] pip list # Tests. - name: Run tests - run: nosetests -v -v ${{ matrix.windows_nose_args }} + run: python -m unittest --verbose # Build and Upload. - name: Build sdist and wheel. + if: success() run: | - pip install wheel - python clean_for_build.py - python setup.py sdist bdist_wheel + pip install flit==3.9.0 + flit --debug build --no-use-vcs - name: Upload sdist and wheel. if: success() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: pyxform--on-${{ matrix.os }}--py${{ matrix.python }} path: ${{ github.workspace }}/dist/pyxform* From 87041a5f4df07e25a1c484fcab4f534274b39a88 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Wed, 17 Jan 2024 20:24:11 +1100 Subject: [PATCH 19/22] dev: add encoding to open() calls for cross-platform test compatibility - either missing before, or mistakenly removed in commit f491ea8f --- pyxform/instance.py | 2 +- pyxform/utils.py | 7 ++++--- pyxform/validators/updater.py | 8 ++++---- pyxform/xls2json.py | 2 +- tests/test_external_instances_for_selects.py | 2 +- tests/test_xform2json.py | 4 ++-- tests/test_xls2json_xls.py | 8 ++++++-- tests/utils.py | 2 +- tests/xform_test_case/test_attribute_columns.py | 4 +++- tests/xform_test_case/test_bugs.py | 12 +++++++++--- 10 files changed, 32 insertions(+), 19 deletions(-) diff --git a/pyxform/instance.py b/pyxform/instance.py index eff6a1cfe..98a600875 100644 --- a/pyxform/instance.py +++ b/pyxform/instance.py @@ -78,7 +78,7 @@ def import_from_xml(self, xml_string_or_filename): import os.path if os.path.isfile(xml_string_or_filename): - xml_str = open(xml_string_or_filename).read() + xml_str = open(xml_string_or_filename, encoding="utf-8").read() else: xml_str = xml_string_or_filename key_val_dict = parse_xform_instance(xml_str) diff --git a/pyxform/utils.py b/pyxform/utils.py index bc555c780..aafb6d352 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -192,7 +192,7 @@ def xls_sheet_to_csv(workbook_path, csv_path, sheet_name): return False if not sheet or sheet.nrows < 2: return False - with open(csv_path, "w", newline="") as f: + with open(csv_path, mode="w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) mask = [v and len(v.strip()) > 0 for v in sheet.row_values(0)] for row_idx in range(sheet.nrows): @@ -220,7 +220,7 @@ def xlsx_sheet_to_csv(workbook_path, csv_path, sheet_name): except KeyError: return False - with open(csv_path, "w", newline="") as f: + with open(csv_path, mode="w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) mask = [not is_empty(cell.value) for cell in sheet[1]] for row in sheet.rows: @@ -260,7 +260,8 @@ def get_languages_with_bad_tags(languages): """ Returns languages with invalid or missing IANA subtags. """ - with open(os.path.join(os.path.dirname(__file__), "iana_subtags.txt")) as f: + path = os.path.join(os.path.dirname(__file__), "iana_subtags.txt") + with open(path, encoding="utf-8") as f: iana_subtags = f.read().splitlines() lang_code_regex = re.compile(r"\((.*)\)$") diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index 732eaa391..6cefc099c 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -96,7 +96,7 @@ def _read_json(file_path): Read the JSON file to a string. """ _UpdateHandler._check_path(file_path=file_path) - with open(file_path) as in_file: + with open(file_path, encoding="utf-8") as in_file: return json.load(in_file) @staticmethod @@ -104,7 +104,7 @@ def _write_json(file_path, content): """ Save the JSON data to a file. """ - with open(file_path, mode="w", newline="\n") as out_file: + with open(file_path, mode="w", encoding="utf-8", newline="\n") as out_file: data = json.dumps(content, indent=2, sort_keys=True) out_file.write(str(data)) @@ -114,7 +114,7 @@ def _read_last_check(file_path): Read the .last_check file. """ _UpdateHandler._check_path(file_path=file_path) - with open(file_path) as in_file: + with open(file_path, encoding="utf-8") as in_file: first_line = in_file.readline() try: last_check = datetime.strptime(first_line, UTC_FMT) @@ -128,7 +128,7 @@ def _write_last_check(file_path, content): """ Write the .last_check file. """ - with open(file_path, mode="w", newline="\n") as out_file: + with open(file_path, mode="w", encoding="utf-8", newline="\n") as out_file: out_file.write(str(content.strftime(UTC_FMT))) @staticmethod diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 5701edba3..3a019fb14 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -1681,7 +1681,7 @@ def __init__(self, path_or_file, default_name=None): def print_warning_log(self, warn_out_file): # Open file to print warning log to. - warn_out = open(warn_out_file, "w") + warn_out = open(warn_out_file, mode="w", encoding="utf-8") warn_out.write("\n".join(self._warnings)) diff --git a/tests/test_external_instances_for_selects.py b/tests/test_external_instances_for_selects.py index b42916757..32b78b280 100644 --- a/tests/test_external_instances_for_selects.py +++ b/tests/test_external_instances_for_selects.py @@ -441,7 +441,7 @@ def test_itemset_csv_generated_from_external_choices(self): self.assertIn(log_msg, [r.message for r in log.records]) self.assertTrue(os.path.exists(itemsets_path)) - with open(itemsets_path) as csv: + with open(itemsets_path, encoding="utf-8") as csv: rows = csv.readlines() # Should have the non-empty headers in the first row. self.assertEqual('"list_name","name","state","city"\n', rows[0]) diff --git a/tests/test_xform2json.py b/tests/test_xform2json.py index 3a2e62ddc..afc1d8d73 100644 --- a/tests/test_xform2json.py +++ b/tests/test_xform2json.py @@ -77,7 +77,7 @@ def test_try_parse_with_path(self): """Should return root node from XML file path.""" xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") self.tidy_file = xml_path - with open(xml_path, "w") as xml_file: + with open(xml_path, mode="w", encoding="utf-8") as xml_file: xml_file.write(self.xml) root = _try_parse(xml_path) self.assertEqual("a", root.tag) @@ -97,7 +97,7 @@ def test_try_parse_with_bad_file(self): """Should raise XMLSyntaxError: file exists but content is not valid.""" xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") self.tidy_file = xml_path - with open(xml_path, "w") as xml_file: + with open(xml_path, mode="w", encoding="utf-8") as xml_file: xml_file.write("not valid xml :(") with self.assertRaises(ParseError): _try_parse(xml_path) diff --git a/tests/test_xls2json_xls.py b/tests/test_xls2json_xls.py index 51b53da08..1f914968a 100644 --- a/tests/test_xls2json_xls.py +++ b/tests/test_xls2json_xls.py @@ -34,7 +34,9 @@ def test_simple_yes_or_no_question(self): with open(output_path, mode="w", encoding="utf-8") as fp: json.dump(x_results, fp=fp, ensure_ascii=False, indent=4) # Compare with the expected output: - with open(expected_output_path) as expected, open(output_path) as observed: + with open(expected_output_path, encoding="utf-8") as expected, open( + output_path, encoding="utf-8" + ) as observed: self.assertEqual(json.load(expected), json.load(observed)) def test_hidden(self): @@ -126,7 +128,9 @@ def test_table(self): with open(output_path, mode="w", encoding="utf-8") as fp: json.dump(x_results, fp=fp, ensure_ascii=False, indent=4) # Compare with the expected output: - with open(expected_output_path) as expected, open(output_path) as observed: + with open(expected_output_path, encoding="utf-8") as expected, open( + output_path, encoding="utf-8" + ) as observed: self.assertEqual(json.load(expected), json.load(observed)) def test_choice_filter_choice_fields(self): diff --git a/tests/utils.py b/tests/utils.py index aec95fe86..b2b2344d4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -95,7 +95,7 @@ def truncate_temp_files(temp_dir): truncate_temp_files(f.path) # Check still in temp directory elif f.path.startswith(temp_root): - with open(f.path, mode="w") as _: + with open(f.path, mode="w", encoding="utf-8") as _: pass diff --git a/tests/xform_test_case/test_attribute_columns.py b/tests/xform_test_case/test_attribute_columns.py index f0e9bcc7f..e560ff3be 100644 --- a/tests/xform_test_case/test_attribute_columns.py +++ b/tests/xform_test_case/test_attribute_columns.py @@ -30,5 +30,7 @@ def test_conversion(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with open(expected_output_path) as expected, open(self.output_path) as observed: + with open(expected_output_path, encoding="utf-8") as expected, open( + self.output_path, encoding="utf-8" + ) as observed: self.assertXFormEqual(expected.read(), observed.read()) diff --git a/tests/xform_test_case/test_bugs.py b/tests/xform_test_case/test_bugs.py index 31091c465..01a7bb7d8 100644 --- a/tests/xform_test_case/test_bugs.py +++ b/tests/xform_test_case/test_bugs.py @@ -93,7 +93,9 @@ def test_conversion(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with open(expected_output_path) as expected, open(self.output_path) as observed: + with open(expected_output_path, encoding="utf-8") as expected, open( + self.output_path, encoding="utf-8" + ) as observed: self.assertXFormEqual(expected.read(), observed.read()) @@ -116,7 +118,9 @@ def test_conversion(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with open(expected_output_path) as expected, open(self.output_path) as observed: + with open(expected_output_path, encoding="utf-8") as expected, open( + self.output_path, encoding="utf-8" + ) as observed: self.assertXFormEqual(expected.read(), observed.read()) @@ -141,7 +145,9 @@ def test_conversion(self): survey.print_xform_to_file(output_path, warnings=warnings) # print warnings # Compare with the expected output: - with open(expected_output_path) as expected, open(output_path) as observed: + with open(expected_output_path, encoding="utf-8") as expected, open( + output_path, encoding="utf-8" + ) as observed: self.assertXFormEqual(expected.read(), observed.read()) From ed0b6a0c32e2e479afb20a52d1dbddef5ba29b4e Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Thu, 8 Feb 2024 20:35:47 +1100 Subject: [PATCH 20/22] dev: linter fixes following rebase onto master --- pyxform/builder.py | 6 ++-- pyxform/utils.py | 6 ++-- tests/pyxform_test_case.py | 50 +++++++++++++++--------------- tests/xform_test_case/test_bugs.py | 2 +- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/pyxform/builder.py b/pyxform/builder.py index 0f3dc2910..a4bc8928b 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -209,18 +209,18 @@ def _add_other_option_to_multiple_choice_question(d: Dict[str, Any]) -> None: @staticmethod def _get_or_other_choice( - choice_list: List[Dict[str, Any]] + choice_list: List[Dict[str, Any]], ) -> Dict[str, Union[str, Dict]]: """ If the choices have any translations, return an OR_OTHER choice for each lang. """ if any(isinstance(c.get(const.LABEL), dict) for c in choice_list): - langs = set( + langs = { lang for c in choice_list for lang in c[const.LABEL] if isinstance(c.get(const.LABEL), dict) - ) + } return { const.NAME: OR_OTHER_CHOICE[const.NAME], const.LABEL: {lang: OR_OTHER_CHOICE[const.LABEL] for lang in langs}, diff --git a/pyxform/utils.py b/pyxform/utils.py index aafb6d352..5c31260fe 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -9,7 +9,7 @@ from json.decoder import JSONDecodeError from typing import Dict, List, NamedTuple, Tuple from xml.dom import Node -from xml.dom.minidom import Element, Text, _write_data, parseString +from xml.dom.minidom import Element, Text, _write_data import openpyxl import xlrd @@ -84,9 +84,9 @@ def writexml(self, writer, indent="", addindent="", newl=""): for cnode in self.childNodes: cnode.writexml(writer, indent + addindent, addindent, newl) writer.write(indent) - writer.write("%s" % (self.tagName, newl)) + writer.write(f"{newl}") else: - writer.write("/>%s" % (newl)) + writer.write(f"/>{newl}") class PatchedText(Text): diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 355c97e4f..31e433db1 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -55,7 +55,7 @@ def md_to_pyxform_survey( id_string: Optional[str] = None, debug: bool = False, autoname: bool = True, - warnings: List[str] = None, + warnings: Optional[List[str]] = None, ): if autoname: kwargs = self._autoname_inputs(name=name, title=title, id_string=id_string) @@ -109,7 +109,7 @@ def _ss_structure_to_pyxform_survey( name: Optional[str] = None, title: Optional[str] = None, id_string: Optional[str] = None, - warnings: List[str] = None, + warnings: Optional[List[str]] = None, ): # using existing methods from the builder imported_survey_json = workbook_to_json( @@ -162,34 +162,34 @@ class PyxformTestCase(PyxformMarkdown, TestCase): def assertPyxformXform( self, # Survey input - md: str = None, - ss_structure: Dict = None, - survey: "Survey" = None, + md: Optional[str] = None, + ss_structure: Optional[Dict] = None, + survey: Optional["Survey"] = None, # XForm assertions - xml__xpath_match: Iterable[str] = None, - xml__xpath_exact: Iterable[Tuple[str, Set[str]]] = None, - xml__xpath_count: Iterable[Tuple[str, int]] = None, + xml__xpath_match: Optional[Iterable[str]] = None, + xml__xpath_exact: Optional[Iterable[Tuple[str, Set[str]]]] = None, + xml__xpath_count: Optional[Iterable[Tuple[str, int]]] = None, # XForm assertions - deprecated - xml__contains: Iterable[str] = None, - xml__excludes: Iterable[str] = None, - model__contains: Iterable[str] = None, - model__excludes: Iterable[str] = None, - itext__contains: Iterable[str] = None, - itext__excludes: Iterable[str] = None, - instance__contains: Iterable[str] = None, + xml__contains: Optional[Iterable[str]] = None, + xml__excludes: Optional[Iterable[str]] = None, + model__contains: Optional[Iterable[str]] = None, + model__excludes: Optional[Iterable[str]] = None, + itext__contains: Optional[Iterable[str]] = None, + itext__excludes: Optional[Iterable[str]] = None, + instance__contains: Optional[Iterable[str]] = None, # Errors assertions - error__contains: Iterable[str] = None, - error__not_contains: Iterable[str] = None, - odk_validate_error__contains: Iterable[str] = None, - warnings__contains: Iterable[str] = None, - warnings__not_contains: Iterable[str] = None, + error__contains: Optional[Iterable[str]] = None, + error__not_contains: Optional[Iterable[str]] = None, + odk_validate_error__contains: Optional[Iterable[str]] = None, + warnings__contains: Optional[Iterable[str]] = None, + warnings__not_contains: Optional[Iterable[str]] = None, warnings_count: Optional[int] = None, errored: bool = False, # Optional extras - name: str = None, - id_string: str = None, - title: str = None, - warnings: List[str] = None, + name: Optional[str] = None, + id_string: Optional[str] = None, + title: Optional[str] = None, + warnings: Optional[List[str]] = None, run_odk_validate: bool = False, debug: bool = False, ): @@ -318,7 +318,7 @@ def _pull_xml_node_from_root(element_selector): "Expected valid survey but compilation failed. Try correcting the " "error with 'debug=True', setting 'errored=True', and or optionally " "'error__contains=[...]'\nError(s): " + "\n".join(errors) - ) + ) from e except ODKValidateError as e: if not odk_validate_error__contains: raise PyxformTestError( diff --git a/tests/xform_test_case/test_bugs.py b/tests/xform_test_case/test_bugs.py index 01a7bb7d8..df627593b 100644 --- a/tests/xform_test_case/test_bugs.py +++ b/tests/xform_test_case/test_bugs.py @@ -155,7 +155,7 @@ class ValidateWrapper(TestCase): maxDiff = None @staticmethod - def test_conversion(self): + def test_conversion(): filename = "ODKValidateWarnings.xlsx" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: From b7c496ab5a96404b75974923252bd607e039e77a Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Thu, 8 Feb 2024 21:03:26 +1100 Subject: [PATCH 21/22] dev: update dev dependencies --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81c3ac671..6861624b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,8 @@ dependencies = [ dev = [ "formencode==2.1.0", # Compare XML "lxml==5.1.0", # XPath test expressions - "psutil==5.9.7", # Process info for performance tests - "ruff==0.1.11", # Format and lint + "psutil==5.9.8", # Process info for performance tests + "ruff==0.2.1", # Format and lint ] [project.urls] @@ -45,7 +45,7 @@ line-length = 90 target-version = "py38" fix = true show-fixes = true -show-source = true +output-format = "full" src = ["pyxform", "tests"] [tool.ruff.lint] From 64903fd1d78be9128c1804bce5e3b241a051a497 Mon Sep 17 00:00:00 2001 From: lindsay stevens Date: Thu, 8 Feb 2024 21:13:18 +1100 Subject: [PATCH 22/22] dev: move binding_conversion dict to aliases, remove irrelevant comment - feedback from PR #685 --- pyxform/aliases.py | 14 ++++++++++++++ pyxform/builder.py | 2 -- pyxform/constants.py | 14 -------------- pyxform/survey_element.py | 5 +++-- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/pyxform/aliases.py b/pyxform/aliases.py index ede93c6fc..a0c416771 100644 --- a/pyxform/aliases.py +++ b/pyxform/aliases.py @@ -180,3 +180,17 @@ "username", ] osm = {"osm": constants.OSM_TYPE} +BINDING_CONVERSIONS = { + "yes": "true()", + "Yes": "true()", + "YES": "true()", + "true": "true()", + "True": "true()", + "TRUE": "true()", + "no": "false()", + "No": "false()", + "NO": "false()", + "false": "false()", + "False": "false()", + "FALSE": "false()", +} diff --git a/pyxform/builder.py b/pyxform/builder.py index a4bc8928b..1b8ecb58c 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -75,8 +75,6 @@ def copy_json_dict(json_dict): class SurveyElementBuilder: - # we use this CLASSES dict to create questions from dictionaries - def __init__(self, **kwargs): # I don't know why we would need an explicit none option for # select alls diff --git a/pyxform/constants.py b/pyxform/constants.py index f3821e44c..0730350eb 100644 --- a/pyxform/constants.py +++ b/pyxform/constants.py @@ -148,20 +148,6 @@ class EntityColumns(StrEnum): "becomes '_setting'." ) -BINDING_CONVERSIONS = { - "yes": "true()", - "Yes": "true()", - "YES": "true()", - "true": "true()", - "True": "true()", - "TRUE": "true()", - "no": "false()", - "No": "false()", - "NO": "false()", - "false": "false()", - "False": "false()", - "FALSE": "false()", -} CONVERTIBLE_BIND_ATTRIBUTES = ( "readonly", "required", diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index 2076e2688..73394aeb2 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -7,6 +7,7 @@ from functools import lru_cache from typing import TYPE_CHECKING, Any, ClassVar, Dict, List +from pyxform import aliases as alias from pyxform import constants as const from pyxform.errors import PyXFormError from pyxform.question_type_dictionary import QUESTION_TYPE_DICT @@ -443,10 +444,10 @@ def xml_bindings(self): # the xls2json side. if ( hashable(v) - and v in const.BINDING_CONVERSIONS + and v in alias.BINDING_CONVERSIONS and k in const.CONVERTIBLE_BIND_ATTRIBUTES ): - v = const.BINDING_CONVERSIONS[v] + v = alias.BINDING_CONVERSIONS[v] if k == "jr:constraintMsg" and ( isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) ):