diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c6a2f43b..a3f9e939b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,38 +12,31 @@ jobs: python: ['3.8'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} # Install dependencies. - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Python cache with dependencies. id: python-cache with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev_requirements.pip') }} + key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }} - name: Install dependencies. run: | python -m pip install --upgrade pip - pip install -r dev_requirements.pip + pip install -e . pip list - # Build. - - name: Build sdist and wheel. - run: | - pip install wheel - python clean_for_build.py - python setup.py sdist bdist_wheel - - # Publish. - - name: Publish release to PyPI with twine + # Build and publish. + - name: Publish release to PyPI if: success() run: | - pip install twine - twine upload dist/pyxform-*-py3-none-any.whl dist/pyxform-*.tar.gz + pip install flit==3.9.0 + flit --debug publish --no-use-vcs env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + FLIT_USERNAME: __token__ + FLIT_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 41ee83f56..f8648b107 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -10,30 +10,28 @@ jobs: python: ['3.8'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} # Install dependencies. - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Python cache with dependencies. id: python-cache with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev_requirements.pip') }} + key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }} - name: Install dependencies. run: | python -m pip install --upgrade pip - pip install -r dev_requirements.pip + pip install -e .[dev] pip list - # Linters. - - run: black pyxform tests --check --diff - - run: isort pyxform tests --check-only --diff - - run: flake8 pyxform tests - - run: pycodestyle pyxform tests + # Linter. + - run: ruff check pyxform tests --no-fix + - run: ruff format pyxform tests --diff test: runs-on: ${{ matrix.os }} @@ -47,38 +45,38 @@ jobs: - os: windows-latest windows_nose_args: --traverse-namespace ./tests steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} # Install dependencies. - - uses: actions/cache@v2 + - uses: actions/cache@v3 name: Python cache with dependencies. id: python-cache with: path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev_requirements.pip') }} + key: ${{ env.pythonLocation }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }} - name: Install dependencies. run: | python -m pip install --upgrade pip - pip install -r dev_requirements.pip + pip install -e .[dev] pip list # Tests. - name: Run tests - run: nosetests -v -v ${{ matrix.windows_nose_args }} + run: python -m unittest --verbose # Build and Upload. - name: Build sdist and wheel. + if: success() run: | - pip install wheel - python clean_for_build.py - python setup.py sdist bdist_wheel + pip install flit==3.9.0 + flit --debug build --no-use-vcs - name: Upload sdist and wheel. if: success() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: pyxform--on-${{ matrix.os }}--py${{ matrix.python }} path: ${{ github.workspace }}/dist/pyxform* diff --git a/README.rst b/README.rst index c3dc2793a..bf988ab96 100644 --- a/README.rst +++ b/README.rst @@ -50,6 +50,7 @@ From the command line, complete the following. These steps use a `virtualenv ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pyxform.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pyxform.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/pyxform" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pyxform" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - make -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index ecd6adc00..000000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,226 +0,0 @@ -# -*- coding: utf-8 -*- -# -# pyxform documentation build configuration file, created by -# sphinx-quickstart on Thu Apr 21 15:01:53 2011. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.autodoc"] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = u"pyxform" -copyright = u"2011, Columbia University, Modi Research Group" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = "0.5" -# The full version, including alpha/beta/rc tags. -release = "0.5" - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "default" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = "pyxformdoc" - - -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ( - "index", - "pyxform.tex", - u"pyxform Documentation", - u"Columbia University, Modi Research Group", - "manual", - ), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ( - "index", - "pyxform", - u"pyxform Documentation", - [u"Columbia University, Modi Research Group"], - 1, - ) -] diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 563c03b3d..000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,54 +0,0 @@ -pyxform -======= - -.. automodule:: pyxform - -xls2xform ---------- - -.. automodule:: pyxform.xls2xform - -API ---- - -pyxform takes an object oriented approach to defining -surveys. Questions, question options, groups of questions, and surveys -are all instances of SurveyElement. This allows us to model a survey -as a tree of SurveyElements. The class inheritance is structured as -follows: - -* SurveyElement - + Option - + Question - - InputQuestion - - UploadQuestion - - MultipleChoiceQuestion - + Section - - RepeatingSection - - GroupedSection - - Survey - -SurveyElement -~~~~~~~~~~~~~ - -.. autoclass:: pyxform.survey_element.SurveyElement - :members: - -Question -~~~~~~~~ - -.. autoclass:: pyxform.question.Question - :members: - -Contents -======== - -.. toctree:: - :maxdepth: 2 - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/pre-commit.sh b/pre-commit.sh index 970d8f7c5..d959c5b4d 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -4,15 +4,6 @@ FILES=$(git diff --cached --name-only --diff-filter=ACM | grep -e '\.py$') if [ -n "$FILES" ]; then - isort -q $FILES -fi - -if [ -n "$FILES" ]; then - if black $FILES; then - touch .commit - fi -fi - -if [ -n "$FILES" ]; then - flake8 $FILES + ruff check pyxform tests + ruff format pyxform tests fi diff --git a/pyproject.toml b/pyproject.toml index 94efc91a7..6861624b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,90 @@ -[tool.black] -# Make sure line length matches setup.cfg setting for pycodestyle and flake8 +[project] +name = "pyxform" +version = "2.0.0" +authors = [ + {name = "github.com/xlsform", email = "info@xlsform.org"}, +] +description = "A Python package to create XForms for ODK Collect." +readme = "README.rst" +requires-python = ">=3.7" +dependencies = [ + "xlrd==2.0.1", # Read XLS files + "openpyxl==3.1.2", # Read XLSX files + "defusedxml==0.7.1", # Parse XML +] + +[project.optional-dependencies] +# Install with `pip install pyxform[dev]`. +dev = [ + "formencode==2.1.0", # Compare XML + "lxml==5.1.0", # XPath test expressions + "psutil==5.9.8", # Process info for performance tests + "ruff==0.2.1", # Format and lint +] + +[project.urls] +Homepage = "https://pypi.python.org/pypi/pyxform/" +Repository = "https://github.com/XLSForm/pyxform/" + +[project.scripts] +xlsform = "pyxform.xls2xform:main_cli" +pyxform_validator_update = "pyxform.validators.updater:main_cli" + +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[tool.flit.module] +name = "pyxform" + +[tool.flit.sdist] +exclude = ["docs", "tests"] + +[tool.ruff] line-length = 90 +target-version = "py38" +fix = true +show-fixes = true +output-format = "full" +src = ["pyxform", "tests"] -[tool.isort] -profile = "black" -multi_line_output = 3 -line_length = 90 -default_section = "THIRDPARTY" -known_first_party = "pyxform" -sections = ['FUTURE', 'STDLIB', 'THIRDPARTY', 'FIRSTPARTY', 'LOCALFOLDER'] +[tool.ruff.lint] +# By default, ruff enables flake8's F rules, along with a subset of the E rules. +select = [ + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "E", # pycodestyle error +# "ERA", # eradicate (commented out code) + "F", # pyflakes + "I", # isort + "PERF", # perflint + "PIE", # flake8-pie + "PL", # pylint +# "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi +# "RET", # flake8-return + "RUF", # ruff-specific rules + "S", # flake8-bandit +# "SIM", # flake8-simplify + "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle warning +] +ignore = [ + "E501", # line-too-long (we have a lot of long strings) + "F821", # undefined-name (doesn't work well with type hints, ruff 0.1.11). + "PERF401", # manual-list-comprehension (false positives on selective transforms) + "PERF402", # manual-list-copy (false positives on selective transforms) + "PLR0911", # too-many-return-statements (complexity not useful to warn every time) + "PLR0912", # too-many-branches (complexity not useful to warn every time) + "PLR0913", # too-many-arguments (complexity not useful to warn every time) + "PLR0915", # too-many-statements (complexity not useful to warn every time) + "PLR2004", # magic-value-comparison (many tests expect certain numbers of things) + "PLW2901", # redefined-loop-name (usually not a bug) + "RUF001", # ambiguous-unicode-character-string (false positives on unicode tests) + "S310", # suspicious-url-open-usage (prone to false positives, ruff 0.1.11) + "S320", # suspicious-xmle-tree-usage (according to defusedxml author lxml is safe) + "S603", # subprocess-without-shell-equals-true (prone to false positives, ruff 0.1.11) + "TRY003", # raise-vanilla-args (reasonable lint but would require large refactor) +] +# per-file-ignores = {"tests/*" = ["E501"]} diff --git a/pyxform/aliases.py b/pyxform/aliases.py index 710c68355..a0c416771 100644 --- a/pyxform/aliases.py +++ b/pyxform/aliases.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Aliases for elements which could mean the same element in XForm but is represented differently on the XLSForm. @@ -181,3 +180,17 @@ "username", ] osm = {"osm": constants.OSM_TYPE} +BINDING_CONVERSIONS = { + "yes": "true()", + "Yes": "true()", + "YES": "true()", + "true": "true()", + "True": "true()", + "TRUE": "true()", + "no": "false()", + "No": "false()", + "NO": "false()", + "false": "false()", + "False": "false()", + "FALSE": "false()", +} diff --git a/pyxform/builder.py b/pyxform/builder.py index 8837593ad..1b8ecb58c 100644 --- a/pyxform/builder.py +++ b/pyxform/builder.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Survey builder functionality. """ @@ -33,6 +32,23 @@ const.NAME: "other", const.LABEL: "Other", } +QUESTION_CLASSES = { + "": Question, + "action": Question, + "input": InputQuestion, + "odk:rank": MultipleChoiceQuestion, + "osm": OsmUploadQuestion, + "range": RangeQuestion, + "select": MultipleChoiceQuestion, + "select1": MultipleChoiceQuestion, + "trigger": TriggerQuestion, + "upload": UploadQuestion, +} +SECTION_CLASSES = { + const.GROUP: GroupedSection, + const.REPEAT: RepeatingSection, + const.SURVEY: Survey, +} def copy_json_dict(json_dict): @@ -42,15 +58,15 @@ def copy_json_dict(json_dict): json_dict_copy = None items = None - if type(json_dict) is list: + if isinstance(json_dict, list): json_dict_copy = [None] * len(json_dict) items = enumerate(json_dict) - elif type(json_dict) is dict: + elif isinstance(json_dict, dict): json_dict_copy = {} items = json_dict.items() for key, value in items: - if type(value) is dict or type(value) is list: + if isinstance(value, (dict, list)): json_dict_copy[key] = copy_json_dict(value) else: json_dict_copy[key] = value @@ -59,26 +75,6 @@ def copy_json_dict(json_dict): class SurveyElementBuilder: - # we use this CLASSES dict to create questions from dictionaries - QUESTION_CLASSES = { - "": Question, - "action": Question, - "input": InputQuestion, - "odk:rank": MultipleChoiceQuestion, - "osm": OsmUploadQuestion, - "range": RangeQuestion, - "select": MultipleChoiceQuestion, - "select1": MultipleChoiceQuestion, - "trigger": TriggerQuestion, - "upload": UploadQuestion, - } - - SECTION_CLASSES = { - const.GROUP: GroupedSection, - const.REPEAT: RepeatingSection, - const.SURVEY: Survey, - } - def __init__(self, **kwargs): # I don't know why we would need an explicit none option for # select alls @@ -97,7 +93,8 @@ def set_sections(self, sections): the name of the section and the value is a dict that can be used to create a whole survey. """ - assert type(sections) == dict + if not isinstance(sections, dict): + raise PyXFormError("""Invalid value for `sections`.""") self._sections = sections def create_survey_element_from_dict( @@ -111,7 +108,7 @@ def create_survey_element_from_dict( if "add_none_option" in d: self._add_none_option = d["add_none_option"] - if d[const.TYPE] in self.SECTION_CLASSES: + if d[const.TYPE] in SECTION_CLASSES: if d[const.TYPE] == const.SURVEY: self._choices = copy.deepcopy(d.get(const.CHOICES, {})) @@ -210,18 +207,18 @@ def _add_other_option_to_multiple_choice_question(d: Dict[str, Any]) -> None: @staticmethod def _get_or_other_choice( - choice_list: List[Dict[str, Any]] + choice_list: List[Dict[str, Any]], ) -> Dict[str, Union[str, Dict]]: """ If the choices have any translations, return an OR_OTHER choice for each lang. """ if any(isinstance(c.get(const.LABEL), dict) for c in choice_list): - langs = set( + langs = { lang for c in choice_list for lang in c[const.LABEL] if isinstance(c.get(const.LABEL), dict) - ) + } return { const.NAME: OR_OTHER_CHOICE[const.NAME], const.LABEL: {lang: OR_OTHER_CHOICE[const.LABEL] for lang in langs}, @@ -257,7 +254,7 @@ def _get_question_class(question_type_str, question_type_dictionary): if control_tag == "upload" and control_dict.get("mediatype") == "osm/*": control_tag = "osm" - return SurveyElementBuilder.QUESTION_CLASSES[control_tag] + return QUESTION_CLASSES[control_tag] @staticmethod def _create_specify_other_question_from_dict(d: Dict[str, Any]) -> InputQuestion: @@ -272,7 +269,7 @@ def _create_specify_other_question_from_dict(d: Dict[str, Any]) -> InputQuestion def _create_section_from_dict(self, d): d_copy = d.copy() children = d_copy.pop(const.CHILDREN, []) - section_class = self.SECTION_CLASSES[d_copy[const.TYPE]] + section_class = SECTION_CLASSES[d_copy[const.TYPE]] if d[const.TYPE] == const.SURVEY and const.TITLE not in d: d_copy[const.TITLE] = d[const.NAME] result = section_class(**d_copy) @@ -336,28 +333,23 @@ def _name_and_label_substitutions(question_template, column_headers): # if the label in column_headers has multiple languages setup a # dictionary by language to do substitutions. info_by_lang = {} - if type(column_headers[const.LABEL]) == dict: - info_by_lang = dict( - [ - ( - lang, - { - const.NAME: column_headers[const.NAME], - const.LABEL: column_headers[const.LABEL][lang], - }, - ) - for lang in column_headers[const.LABEL].keys() - ] - ) + if isinstance(column_headers[const.LABEL], dict): + info_by_lang = { + lang: { + const.NAME: column_headers[const.NAME], + const.LABEL: column_headers[const.LABEL][lang], + } + for lang in column_headers[const.LABEL].keys() + } result = question_template.copy() for key in result.keys(): - if type(result[key]) == str: + if isinstance(result[key], str): result[key] %= column_headers - elif type(result[key]) == dict: + elif isinstance(result[key], dict): result[key] = result[key].copy() for key2 in result[key].keys(): - if type(column_headers[const.LABEL]) == dict: + if isinstance(column_headers[const.LABEL], dict): result[key][key2] %= info_by_lang.get(key2, column_headers) else: result[key][key2] %= column_headers @@ -394,9 +386,9 @@ def create_survey_from_xls(path_or_file, default_name=None): def create_survey( - name_of_main_section: str = None, - sections: Dict[str, Dict] = None, - main_section: Dict[str, Any] = None, + name_of_main_section: Optional[str] = None, + sections: Optional[Dict[str, Dict]] = None, + main_section: Optional[Dict[str, Any]] = None, id_string: Optional[str] = None, title: Optional[str] = None, ) -> Survey: diff --git a/pyxform/constants.py b/pyxform/constants.py index 792072c10..0730350eb 100644 --- a/pyxform/constants.py +++ b/pyxform/constants.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ This file contains constants that correspond with the property names in the json survey format. (@see json_form_schema.json) These names are to be shared @@ -42,9 +41,7 @@ ATTRIBUTE = "attribute" ALLOW_CHOICE_DUPLICATES = "allow_choice_duplicates" -BIND = ( - "bind" # TODO: What should I do with the nested types? (readonly and relevant) # noqa -) +BIND = "bind" # TODO: What should I do with the nested types? (readonly and relevant) MEDIA = "media" CONTROL = "control" APPEARANCE = "appearance" @@ -72,7 +69,7 @@ # XLS Specific constants LIST_NAME = "list name" CASCADING_SELECT = "cascading_select" -TABLE_LIST = "table-list" # hyphenated because it goes in appearance, and convention for appearance column is dashes # noqa +TABLE_LIST = "table-list" # hyphenated because it goes in appearance, and convention for appearance column is dashes FIELD_LIST = "field-list" LIST_NOLABEL = "list-nolabel" @@ -150,3 +147,11 @@ class EntityColumns(StrEnum): "prefix the sheet name with an underscore. For example 'setting' " "becomes '_setting'." ) + +CONVERTIBLE_BIND_ATTRIBUTES = ( + "readonly", + "required", + "relevant", + "constraint", + "calculate", +) diff --git a/pyxform/entities/entities_parsing.py b/pyxform/entities/entities_parsing.py index 85a758441..948d61db8 100644 --- a/pyxform/entities/entities_parsing.py +++ b/pyxform/entities/entities_parsing.py @@ -127,7 +127,7 @@ def validate_entity_saveto( def validate_entities_columns(row: Dict): extra = {k: None for k in row.keys() if k not in EC.value_list()} if 0 < len(extra): - fmt_extra = ", ".join((f"'{k}'" for k in extra.keys())) + fmt_extra = ", ".join(f"'{k}'" for k in extra.keys()) msg = ( f"The entities sheet included the following unexpected column(s): {fmt_extra}. " f"These columns are not supported by this version of pyxform. Please either: " diff --git a/pyxform/entities/entity_declaration.py b/pyxform/entities/entity_declaration.py index 33ee5ff14..cd85991f5 100644 --- a/pyxform/entities/entity_declaration.py +++ b/pyxform/entities/entity_declaration.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from pyxform import constants as const from pyxform.survey_element import SurveyElement from pyxform.utils import node diff --git a/pyxform/errors.py b/pyxform/errors.py index 606b50707..51aaf3845 100644 --- a/pyxform/errors.py +++ b/pyxform/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Common base classes for pyxform exceptions. """ @@ -7,10 +6,6 @@ class PyXFormError(Exception): """Common base class for pyxform exceptions.""" - pass - class ValidationError(PyXFormError): """Common base class for pyxform validation exceptions.""" - - pass diff --git a/pyxform/external_instance.py b/pyxform/external_instance.py index 1f0838037..c90f06c20 100644 --- a/pyxform/external_instance.py +++ b/pyxform/external_instance.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ ExternalInstance class module """ @@ -12,4 +11,3 @@ def xml_control(self): Exists here because there's a soft abstractmethod in SurveyElement. """ - pass diff --git a/pyxform/file_utils.py b/pyxform/file_utils.py index c827e6514..862ef115f 100644 --- a/pyxform/file_utils.py +++ b/pyxform/file_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ The pyxform file utility functions. """ diff --git a/pyxform/instance.py b/pyxform/instance.py index b4ab9a17c..98a600875 100644 --- a/pyxform/instance.py +++ b/pyxform/instance.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- """ SurveyInstance class module. """ +from pyxform.errors import PyXFormError from pyxform.xform_instance_parser import parse_xform_instance @@ -35,7 +35,7 @@ def xpaths(self): def answer(self, name=None, value=None): if name is None: - raise Exception("In answering, name must be given") + raise PyXFormError("In answering, name must be given") # ahh. this is horrible, but we need the xpath dict in survey # to be up-to-date ...maybe @@ -57,11 +57,11 @@ def to_xml(self): A horrible way to do this, but it works (until we need the attributes pumped out in order, etc) """ - open_str = """<%s id="%s">""" % (self._name, self._id) + open_str = f"""<{self._name} id="{self._id}">""" close_str = """""" % self._name vals = "" for k, v in self._answers.items(): - vals += "<%s>%s" % (k, str(v), k) + vals += f"<{k}>{v!s}" output = open_str + vals + close_str return output @@ -78,7 +78,7 @@ def import_from_xml(self, xml_string_or_filename): import os.path if os.path.isfile(xml_string_or_filename): - xml_str = open(xml_string_or_filename).read() + xml_str = open(xml_string_or_filename, encoding="utf-8").read() else: xml_str = xml_string_or_filename key_val_dict = parse_xform_instance(xml_str) diff --git a/pyxform/parsing/instance_expression.py b/pyxform/parsing/instance_expression.py index 1118c0211..9b908e0f8 100644 --- a/pyxform/parsing/instance_expression.py +++ b/pyxform/parsing/instance_expression.py @@ -91,7 +91,7 @@ def find_boundaries(xml_text: str) -> List[Tuple[int, int]]: # Pair up the boundaries [1, 2, 3, 4] -> [(1, 2), (3, 4)]. bounds = iter(boundaries) - pos_bounds = [(x, y) for x, y in zip(bounds, bounds)] + pos_bounds = list(zip(bounds, bounds)) return pos_bounds diff --git a/pyxform/question.py b/pyxform/question.py index 9fa5600e3..684b40976 100644 --- a/pyxform/question.py +++ b/pyxform/question.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ XForm Survey element classes for different question types. """ @@ -54,8 +53,9 @@ def xml_control(self): if nested_setvalues: for setvalue in nested_setvalues: msg = ( - "The question ${%s} is not user-visible so it can't be used as a calculation trigger for question ${%s}." - % (self.name, setvalue[0]) + f"The question ${{{self.name}}} is not user-visible " + "so it can't be used as a calculation trigger for " + f"question ${{{setvalue[0]}}}." ) raise PyXFormError(msg) return None @@ -200,7 +200,8 @@ def validate(self): choice.validate() def build_xml(self): - assert self.bind["type"] in ["string", "odk:rank"] + if self.bind["type"] not in ["string", "odk:rank"]: + raise PyXFormError("""Invalid value for `self.bind["type"]`.""") survey = self.get_root() control_dict = self.control.copy() # Resolve field references in attributes @@ -239,13 +240,12 @@ def build_xml(self): ) if file_extension in EXTERNAL_INSTANCE_EXTENSIONS: - itemset = itemset + pass + elif not multi_language and not has_media and not has_dyn_label: + itemset = self["itemset"] else: - if not multi_language and not has_media and not has_dyn_label: - itemset = self["itemset"] - else: - itemset = self["itemset"] - itemset_label_ref = "jr:itext(itextId)" + itemset = self["itemset"] + itemset_label_ref = "jr:itext(itextId)" choice_filter = survey.insert_xpaths( choice_filter, self, True, is_previous_question @@ -306,7 +306,7 @@ def build_xml(self): class SelectOneQuestion(MultipleChoiceQuestion): def __init__(self, **kwargs): - super(SelectOneQuestion, self).__init__(**kwargs) + super().__init__(**kwargs) self._dict[self.TYPE] = "select one" @@ -315,7 +315,7 @@ def __init__(self, **kwargs): kwargs_copy = kwargs.copy() choices = kwargs_copy.pop("choices", []) + kwargs_copy.pop("children", []) - super(Tag, self).__init__(**kwargs_copy) + super().__init__(**kwargs_copy) if choices: self.children = [] @@ -345,7 +345,7 @@ def __init__(self, **kwargs): kwargs_copy = kwargs.copy() tags = kwargs_copy.pop("tags", []) + kwargs_copy.pop("children", []) - super(OsmUploadQuestion, self).__init__(**kwargs_copy) + super().__init__(**kwargs_copy) if tags: self.children = [] diff --git a/pyxform/question_type_dictionary.py b/pyxform/question_type_dictionary.py index 43be6cdfe..84f35fbf5 100644 --- a/pyxform/question_type_dictionary.py +++ b/pyxform/question_type_dictionary.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ XForm survey question type mapping dictionary module. """ diff --git a/pyxform/section.py b/pyxform/section.py index ddc44a8d0..30f467cce 100644 --- a/pyxform/section.py +++ b/pyxform/section.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Section survey element module. """ @@ -10,7 +9,7 @@ class Section(SurveyElement): def validate(self): - super(Section, self).validate() + super().validate() for element in self.children: element.validate() self._validate_uniqueness_of_element_names() @@ -23,9 +22,8 @@ def _validate_uniqueness_of_element_names(self): elem_lower = element.name.lower() if elem_lower in element_slugs: raise PyXFormError( - "There are more than one survey elements named '%s' " - "(case-insensitive) in the section named '%s'." - % (elem_lower, self.name) + f"There are more than one survey elements named '{elem_lower}' " + f"(case-insensitive) in the section named '{self.name}'." ) element_slugs.add(elem_lower) @@ -77,8 +75,7 @@ def xml_instance_array(self): """ for child in self.children: if child.get("flat"): - for grandchild in child.xml_instance_array(): - yield grandchild + yield from child.xml_instance_array() else: yield child.xml_instance() @@ -145,7 +142,7 @@ def _dynamic_defaults_helper(self, current, nodes): # I'm anal about matching function signatures when overriding a function, # but there's no reason for kwargs to be an argument def template_instance(self, **kwargs): - return super(RepeatingSection, self).generate_repeating_template(**kwargs) + return super().generate_repeating_template(**kwargs) class GroupedSection(Section): @@ -197,6 +194,6 @@ def xml_control(self): def to_json_dict(self): # This is quite hacky, might want to think about a smart way # to approach this problem. - result = super(GroupedSection, self).to_json_dict() + result = super().to_json_dict() result["type"] = "group" return result diff --git a/pyxform/survey.py b/pyxform/survey.py index a45659b1f..22b237f29 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- """ Survey module with XForm Survey objects and utility functions. """ -import codecs import os import re import tempfile @@ -223,7 +221,7 @@ class Survey(Section): def validate(self): if self.id_string in [None, "None"]: raise PyXFormError("Survey cannot have an empty id_string") - super(Survey, self).validate() + super().validate() self._validate_uniqueness_of_section_names() def _validate_uniqueness_of_section_names(self): @@ -261,13 +259,11 @@ def get_nsmap(self): xmlns = "xmlns:" nsmap = NSMAP.copy() nsmap.update( - dict( - [ - (xmlns + k, v.replace('"', "").replace("'", "")) - for k, v in nslist - if xmlns + k not in nsmap - ] - ) + { + xmlns + k: v.replace('"', "").replace("'", "") + for k, v in nslist + if xmlns + k not in nsmap + } ) return nsmap @@ -358,7 +354,7 @@ def _generate_external_instances(element) -> Optional[InstanceInfo]: name = element["name"] extension = element["type"].split("-")[0] prefix = "file-csv" if extension == "csv" else "file" - src = "jr://{}/{}.{}".format(prefix, name, extension) + src = f"jr://{prefix}/{name}.{extension}" return InstanceInfo( type="external", context="[type: {t}, name: {n}]".format( @@ -392,10 +388,8 @@ def _validate_external_instances(instances) -> None: contexts = ", ".join(x.context for x in copies) errors.append( "Instance names must be unique within a form. " - "The name '{i}' was found {c} time(s), " - "under these contexts: {contexts}".format( - i=element, c=len(copies), contexts=contexts - ) + f"The name '{element}' was found {len(copies)} time(s), " + f"under these contexts: {contexts}" ) if errors: raise ValidationError("\n".join(errors)) @@ -422,7 +416,7 @@ def get_pulldata_functions(element): return functions_present def get_instance_info(element, file_id): - uri = "jr://file-csv/{}.csv".format(file_id) + uri = f"jr://file-csv/{file_id}.csv" return InstanceInfo( type="pulldata", @@ -455,10 +449,8 @@ def _generate_from_file_instances(element) -> Optional[InstanceInfo]: itemset = element.get("itemset") file_id, ext = os.path.splitext(itemset) if itemset and ext in EXTERNAL_INSTANCE_EXTENSIONS: - uri = "jr://%s/%s" % ( - "file" if ext == ".xml" or ext == ".geojson" else "file-%s" % ext[1:], - itemset, - ) + file_ext = "file" if ext in {".xml", ".geojson"} else f"file-{ext[1:]}" + uri = f"jr://{file_ext}/{itemset}" return InstanceInfo( type="file", context="[type: {t}, name: {n}]".format( @@ -569,16 +561,9 @@ def _generate_instances(self) -> Iterator[DetachableElement]: msg = ( "The same instance id will be generated for different " "external instance source URIs. Please check the form." - " Instance name: '{i}', Existing type: '{e}', " - "Existing URI: '{iu}', Duplicate type: '{d}', " - "Duplicate URI: '{du}', Duplicate context: '{c}'.".format( - i=i.name, - iu=seen[i.name].src, - e=seen[i.name].type, - d=i.type, - du=i.src, - c=i.context, - ) + f" Instance name: '{i.name}', Existing type: '{seen[i.name].type}', " + f"Existing URI: '{seen[i.name].src}', Duplicate type: '{i.type}', " + f"Duplicate URI: '{i.src}', Duplicate context: '{i.context}'." ) raise PyXFormError(msg) elif i.name in seen.keys() and seen[i.name].src == i.src: @@ -619,7 +604,7 @@ def xml_model(self): model_children += self.xml_actions() if self.submission_url or self.public_key or self.auto_send or self.auto_delete: - submission_attrs = dict() + submission_attrs = {} if self.submission_url: submission_attrs["action"] = self.submission_url submission_attrs["method"] = "post" @@ -708,15 +693,14 @@ def _setup_translations(self): def _setup_choice_translations( name, choice_value, itext_id ) -> Generator[Tuple[List[str], str], None, None]: - for media_or_lang, value in choice_value.items(): # noqa + for media_or_lang, value in choice_value.items(): if isinstance(value, dict): for language, val in value.items(): yield ([language, itext_id, media_or_lang], val) + elif name == constants.MEDIA: + yield ([self.default_language, itext_id, media_or_lang], value) else: - if name == constants.MEDIA: - yield ([self.default_language, itext_id, media_or_lang], value) - else: - yield ([media_or_lang, itext_id, "long"], value) + yield ([media_or_lang, itext_id, "long"], value) itemsets_multi_language = set() itemsets_has_media = set() @@ -803,11 +787,11 @@ def _add_empty_translations(self): This disables any of the default_language fallback functionality. """ paths = {} - for lang, translation in self._translations.items(): + for translation in self._translations.values(): for path, content in translation.items(): paths[path] = paths.get(path, set()).union(content.keys()) - for lang, translation in self._translations.items(): + for lang in self._translations: for path, content_types in paths.items(): if path not in self._translations[lang]: self._translations[lang][path] = {} @@ -835,7 +819,6 @@ def _set_up_media_translations(media_dict, translation_key): media_dict = media_dict_default for media_type, possibly_localized_media in media_dict.items(): - if media_type not in SurveyElement.SUPPORTED_MEDIA: raise PyXFormError("Media type: " + media_type + " not supported") @@ -848,7 +831,6 @@ def _set_up_media_translations(media_dict, translation_key): localized_media = {self.default_language: possibly_localized_media} for language, media in localized_media.items(): - # Create the required dictionaries in _translations, # then add media as a leaf value: if language not in self._translations: @@ -895,7 +877,7 @@ def itext(self): label_type = label_name.partition(":")[-1] if not isinstance(content, dict): - raise Exception() + raise PyXFormError("""Invalid value for `content`.""") for media_type, media_value in content.items(): # Ignore key indicating Question or Choice translation type. @@ -930,7 +912,7 @@ def itext(self): itext_nodes.append( node("value", value, toParseString=output_inserted) ) - elif media_type == "image" or media_type == "big-image": + elif media_type in {"image", "big-image"}: if value != "-": itext_nodes.append( node( @@ -940,16 +922,15 @@ def itext(self): toParseString=output_inserted, ) ) - else: - if value != "-": - itext_nodes.append( - node( - "value", - "jr://" + media_type + "/" + value, - form=media_type, - toParseString=output_inserted, - ) + elif value != "-": + itext_nodes.append( + node( + "value", + "jr://" + media_type + "/" + value, + form=media_type, + toParseString=output_inserted, ) + ) result[-1].appendChild(node("text", *itext_nodes, id=label_name)) @@ -1039,14 +1020,12 @@ def _is_return_relative_path() -> bool: current_matchobj = matchobj if not last_saved and context: - if not is_indexed_repeat: return True # It is possible to have multiple indexed-repeat in an expression indexed_repeats_iter = RE_INDEXED_REPEAT.finditer(matchobj.string) for indexed_repeat in indexed_repeats_iter: - # Make sure current ${name} is in the correct indexed-repeat if current_matchobj.end() > indexed_repeat.end(): try: @@ -1068,7 +1047,7 @@ def _is_return_relative_path() -> bool: .group(1) .split(",") ) - name_arg = "${{{0}}}".format(name) + name_arg = f"${{{name}}}" for idx, arg in enumerate(indexed_repeat_args): if name_arg in arg.strip(): indexed_repeat_name_index = idx @@ -1082,8 +1061,8 @@ def _is_return_relative_path() -> bool: return False intro = ( - "There has been a problem trying to replace %s with the " - "XPath to the survey element named '%s'." % (matchobj.group(0), name) + f"There has been a problem trying to replace {matchobj.group(0)} with the " + f"XPath to the survey element named '{name}'." ) if name not in self._xpath: raise PyXFormError(intro + " There is no survey element with this name.") @@ -1175,15 +1154,15 @@ def print_xform_to_file( if not path: path = self._print_name + ".xml" try: - with codecs.open(path, mode="w", encoding="utf-8") as file_obj: + with open(path, mode="w", encoding="utf-8") as file_obj: if pretty_print: file_obj.write(self._to_pretty_xml()) else: file_obj.write(self._to_ugly_xml()) - except Exception as error: + except Exception: if os.path.exists(path): os.unlink(path) - raise error + raise if validate: warnings.extend(odk_validate.check_xform(path)) if enketo: @@ -1195,7 +1174,7 @@ def print_xform_to_file( bad_languages = get_languages_with_bad_tags(translations) if bad_languages: warnings.append( - "\tThe following language declarations do not contain " + "The following language declarations do not contain " "valid machine-readable codes: " + ", ".join(bad_languages) + ". " diff --git a/pyxform/survey_element.py b/pyxform/survey_element.py index 9a11508ea..73394aeb2 100644 --- a/pyxform/survey_element.py +++ b/pyxform/survey_element.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Survey Element base class for all survey elements. """ @@ -6,9 +5,10 @@ import re from collections import deque from functools import lru_cache -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List -from pyxform import constants +from pyxform import aliases as alias +from pyxform import constants as const from pyxform.errors import PyXFormError from pyxform.question_type_dictionary import QUESTION_TYPE_DICT from pyxform.utils import ( @@ -21,19 +21,59 @@ from pyxform.xlsparseutils import is_valid_xml_tag if TYPE_CHECKING: - from typing import List - from pyxform.utils import DetachableElement +# The following are important keys for the underlying dict that describes SurveyElement +FIELDS = { + "name": str, + const.COMPACT_TAG: str, # used for compact (sms) representation + "sms_field": str, + "sms_option": str, + "label": str, + "hint": str, + "guidance_hint": str, + "default": str, + "type": str, + "appearance": str, + "parameters": dict, + "intent": str, + "jr:count": str, + "bind": dict, + "instance": dict, + "control": dict, + "media": dict, + # this node will also have a parent and children, like a tree! + "parent": lambda: None, + "children": list, + "itemset": str, + "choice_filter": str, + "query": str, + "autoplay": str, + "flat": lambda: False, + "action": str, + "list_name": str, + "trigger": str, +} + def _overlay(over, under): - if type(under) == dict: + if isinstance(under, dict): result = under.copy() result.update(over) return result return over if over else under +@lru_cache(maxsize=65536) +def any_repeat(survey_element: "SurveyElement", parent_xpath: str) -> bool: + """Return True if there ia any repeat in `parent_xpath`.""" + for item in survey_element.iter_descendants(): + if item.get_xpath() == parent_xpath and item.type == const.REPEAT: + return True + + return False + + class SurveyElement(dict): """ SurveyElement is the base class we'll looks for the following keys @@ -41,38 +81,8 @@ class SurveyElement(dict): children, and question_type_dictionary. """ - # the following are important keys for the underlying dict that - # describes this survey element - FIELDS = { - "name": str, - constants.COMPACT_TAG: str, # used for compact (sms) representation - "sms_field": str, - "sms_option": str, - "label": str, - "hint": str, - "guidance_hint": str, - "default": str, - "type": str, - "appearance": str, - "parameters": dict, - "intent": str, - "jr:count": str, - "bind": dict, - "instance": dict, - "control": dict, - "media": dict, - # this node will also have a parent and children, like a tree! - "parent": lambda: None, - "children": list, - "itemset": str, - "choice_filter": str, - "query": str, - "autoplay": str, - "flat": lambda: False, - "action": str, - "list_name": str, - "trigger": str, - } + __name__ = "SurveyElement" + FIELDS: ClassVar[Dict[str, Any]] = FIELDS.copy() def _default(self): # TODO: need way to override question type dictionary @@ -95,10 +105,6 @@ def __getattr__(self, key): def __hash__(self): return hash(id(self)) - @property - def __name__(self): - return "SurveyElement" - def __setattr__(self, key, value): self[key] = value @@ -123,35 +129,12 @@ def add_child(self, child): child.parent = self def add_children(self, children): - if type(children) == list: + if isinstance(children, list): for child in children: self.add_child(child) else: self.add_child(children) - BINDING_CONVERSIONS = { - "yes": "true()", - "Yes": "true()", - "YES": "true()", - "true": "true()", - "True": "true()", - "TRUE": "true()", - "no": "false()", - "No": "false()", - "NO": "false()", - "false": "false()", - "False": "false()", - "FALSE": "false()", - } - - CONVERTIBLE_BIND_ATTRIBUTES = ( - "readonly", - "required", - "relevant", - "constraint", - "calculate", - ) - # Supported media types for attaching to questions SUPPORTED_MEDIA = ("image", "big-image", "audio", "video") @@ -159,7 +142,7 @@ def validate(self): if not is_valid_xml_tag(self.name): invalid_char = re.search(INVALID_XFORM_TAG_REGEXP, self.name) raise PyXFormError( - f"The name '{self.name}' contains an invalid character '{invalid_char.group(0)}'. Names {constants.XML_IDENTIFIER_ERROR_MESSAGE}" + f"The name '{self.name}' contains an invalid character '{invalid_char.group(0)}'. Names {const.XML_IDENTIFIER_ERROR_MESSAGE}" ) # TODO: Make sure renaming this doesn't cause any problems @@ -173,17 +156,11 @@ def iter_descendants(self): # it really seems like this method should not yield self yield self for e in self.children: - for f in e.iter_descendants(): - yield f + yield from e.iter_descendants() - @lru_cache(maxsize=None) - def any_repeat(self, parent_xpath): + def any_repeat(self, parent_xpath: str) -> bool: """Return True if there ia any repeat in `parent_xpath`.""" - for item in self.iter_descendants(): - if item.get_xpath() == parent_xpath and item.type == constants.REPEAT: - return True - - return False + return any_repeat(survey_element=self, parent_xpath=parent_xpath) def get_lineage(self): """ @@ -282,9 +259,9 @@ def get_translations(self, default_language): the block. @see survey._setup_translations """ bind_dict = self.get("bind") - if bind_dict and type(bind_dict) is dict: + if bind_dict and isinstance(bind_dict, dict): constraint_msg = bind_dict.get("jr:constraintMsg") - if type(constraint_msg) is dict: + if isinstance(constraint_msg, dict): for lang, text in constraint_msg.items(): yield { "path": self._translation_path("jr:constraintMsg"), @@ -301,7 +278,7 @@ def get_translations(self, default_language): } required_msg = bind_dict.get("jr:requiredMsg") - if type(required_msg) is dict: + if isinstance(required_msg, dict): for lang, text in required_msg.items(): yield { "path": self._translation_path("jr:requiredMsg"), @@ -317,7 +294,7 @@ def get_translations(self, default_language): "output_context": self, } no_app_error_string = bind_dict.get("jr:noAppErrorString") - if type(no_app_error_string) is dict: + if isinstance(no_app_error_string, dict): for lang, text in no_app_error_string.items(): yield { "path": self._translation_path("jr:noAppErrorString"), @@ -332,7 +309,7 @@ def get_translations(self, default_language): if ( display_element == "label" and self.needs_itext_ref() - and type(label_or_hint) is not dict + and not isinstance(label_or_hint, dict) and label_or_hint ): label_or_hint = {default_language: label_or_hint} @@ -356,7 +333,7 @@ def get_translations(self, default_language): ): label_or_hint = {default_language: label_or_hint} - if type(label_or_hint) is dict: + if isinstance(label_or_hint, dict): for lang, text in label_or_hint.items(): yield { "display_element": display_element, # Not used @@ -375,8 +352,8 @@ def get_media_keys(self): return {"media": "%s:media" % self.get_xpath()} def needs_itext_ref(self): - return type(self.label) is dict or ( - type(self.media) is dict and len(self.media) > 0 + return isinstance(self.label, dict) or ( + isinstance(self.media, dict) and len(self.media) > 0 ) def get_setvalue_node_for_dynamic_default(self, in_repeat=False): @@ -416,7 +393,7 @@ def xml_hint(self): hint, output_inserted = self.get_root().insert_output_values(self.hint, self) return node("hint", hint, toParseString=output_inserted) - def xml_label_and_hint(self) -> "List[DetachableElement]": + def xml_label_and_hint(self) -> List["DetachableElement"]: """ Return a list containing one node for the label and if there is a hint one node for the hint. @@ -467,19 +444,19 @@ def xml_bindings(self): # the xls2json side. if ( hashable(v) - and v in self.BINDING_CONVERSIONS - and k in self.CONVERTIBLE_BIND_ATTRIBUTES + and v in alias.BINDING_CONVERSIONS + and k in const.CONVERTIBLE_BIND_ATTRIBUTES ): - v = self.BINDING_CONVERSIONS[v] + v = alias.BINDING_CONVERSIONS[v] if k == "jr:constraintMsg" and ( - type(v) is dict or re.search(BRACKETED_TAG_REGEX, v) + isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) ): v = "jr:itext('%s')" % self._translation_path("jr:constraintMsg") if k == "jr:requiredMsg" and ( - type(v) is dict or re.search(BRACKETED_TAG_REGEX, v) + isinstance(v, dict) or re.search(BRACKETED_TAG_REGEX, v) ): v = "jr:itext('%s')" % self._translation_path("jr:requiredMsg") - if k == "jr:noAppErrorString" and type(v) is dict: + if k == "jr:noAppErrorString" and isinstance(v, dict): v = "jr:itext('%s')" % self._translation_path("jr:noAppErrorString") bind_dict[k] = survey.insert_xpaths(v, context=self) return [node("bind", nodeset=self.get_xpath(), **bind_dict)] diff --git a/pyxform/translator.py b/pyxform/translator.py index df79611af..e85564963 100644 --- a/pyxform/translator.py +++ b/pyxform/translator.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Translator class module. """ diff --git a/pyxform/util/enum.py b/pyxform/util/enum.py index 4870013a4..86d62976c 100644 --- a/pyxform/util/enum.py +++ b/pyxform/util/enum.py @@ -9,19 +9,19 @@ class StrEnum(str, Enum): def __new__(cls, *values): "values must already be of type `str`" if len(values) > 3: - raise TypeError("too many arguments for str(): %r" % (values,)) + raise TypeError(f"too many arguments for str(): {values!r}") if len(values) == 1: # it must be a string if not isinstance(values[0], str): - raise TypeError("%r is not a string" % (values[0],)) + raise TypeError(f"{values[0]!r} is not a string") if len(values) >= 2: # check that encoding argument is a string if not isinstance(values[1], str): - raise TypeError("encoding must be a string, not %r" % (values[1],)) + raise TypeError(f"encoding must be a string, not {values[1]!r}") if len(values) == 3: # check that errors argument is a string if not isinstance(values[2], str): - raise TypeError("errors must be a string, not %r" % (values[2])) + raise TypeError(f"errors must be a string, not {values[2]!r}") value = str(*values) member = str.__new__(cls, value) member._value_ = value diff --git a/pyxform/utils.py b/pyxform/utils.py index 70bb194d8..5c31260fe 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -1,22 +1,21 @@ -# -*- coding: utf-8 -*- """ pyxform utils module. """ -import codecs import copy import csv import json import os import re -from collections import namedtuple from json.decoder import JSONDecodeError -from typing import Dict, List, Tuple +from typing import Dict, List, NamedTuple, Tuple from xml.dom import Node -from xml.dom.minidom import Element, Text, _write_data, parseString +from xml.dom.minidom import Element, Text, _write_data import openpyxl import xlrd +from defusedxml.minidom import parseString +from pyxform.errors import PyXFormError from pyxform.xls2json_backends import is_empty, xls_value_to_unicode, xlsx_value_to_str SEP = "_" @@ -85,15 +84,15 @@ def writexml(self, writer, indent="", addindent="", newl=""): for cnode in self.childNodes: cnode.writexml(writer, indent + addindent, addindent, newl) writer.write(indent) - writer.write("%s" % (self.tagName, newl)) + writer.write(f"{newl}") else: - writer.write("/>%s" % (newl)) + writer.write(f"/>{newl}") class PatchedText(Text): def writexml(self, writer, indent="", addindent="", newl=""): """Same as original but no replacing double quotes with '"'.""" - data = "%s%s%s" % (indent, self.data, newl) + data = "".join((indent, self.data, newl)) if data: data = data.replace("&", "&").replace("<", "<").replace(">", ">") writer.write(data) @@ -111,8 +110,9 @@ def node(*args, **kwargs) -> DetachableElement: tag = args[0] if len(args) > 0 else kwargs["tag"] args = args[1:] result = DetachableElement(tag) - unicode_args = [u for u in args if type(u) == str] - assert len(unicode_args) <= 1 + unicode_args = [u for u in args if isinstance(u, str)] + if len(unicode_args) > 1: + raise PyXFormError("""Invalid value for `unicode_args`.""") parsed_string = False # Convert the kwargs xml attribute dictionary to a xml.dom.minidom.Element. @@ -147,11 +147,11 @@ def node(*args, **kwargs) -> DetachableElement: text_node.data = unicode_args[0] result.appendChild(text_node) for n in args: - if type(n) == int or type(n) == float or type(n) == bytes: + if isinstance(n, (int, float, bytes)): text_node = PatchedText() text_node.data = str(n) result.appendChild(text_node) - elif type(n) is not str: + elif not isinstance(n, str): result.appendChild(n) return result @@ -164,9 +164,9 @@ def get_pyobj_from_json(str_or_path): """ try: # see if treating str_or_path as a path works - fp = codecs.open(str_or_path, mode="r", encoding="utf-8") - doc = json.load(fp) - except (IOError, JSONDecodeError, OSError): + with open(str_or_path, encoding="utf-8") as fp: + doc = json.load(fp) + except (JSONDecodeError, OSError): # if it doesn't work load the text doc = json.loads(str_or_path) return doc @@ -174,8 +174,7 @@ def get_pyobj_from_json(str_or_path): def flatten(li): for subli in li: - for it in subli: - yield it + yield from subli def sheet_to_csv(workbook_path, csv_path, sheet_name): @@ -193,7 +192,7 @@ def xls_sheet_to_csv(workbook_path, csv_path, sheet_name): return False if not sheet or sheet.nrows < 2: return False - with open(csv_path, "w", newline="") as f: + with open(csv_path, mode="w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) mask = [v and len(v.strip()) > 0 for v in sheet.row_values(0)] for row_idx in range(sheet.nrows): @@ -221,7 +220,7 @@ def xlsx_sheet_to_csv(workbook_path, csv_path, sheet_name): except KeyError: return False - with open(csv_path, "w", newline="") as f: + with open(csv_path, mode="w", encoding="utf-8", newline="") as f: writer = csv.writer(f, quoting=csv.QUOTE_ALL) mask = [not is_empty(cell.value) for cell in sheet[1]] for row in sheet.rows: @@ -261,7 +260,8 @@ def get_languages_with_bad_tags(languages): """ Returns languages with invalid or missing IANA subtags. """ - with open(os.path.join(os.path.dirname(__file__), "iana_subtags.txt")) as f: + path = os.path.join(os.path.dirname(__file__), "iana_subtags.txt") + with open(path, encoding="utf-8") as f: iana_subtags = f.read().splitlines() lang_code_regex = re.compile(r"\((.*)\)$") @@ -271,7 +271,7 @@ def get_languages_with_bad_tags(languages): lang_code = re.search(lang_code_regex, lang) if lang != "default" and ( - not lang_code or not lang_code.group(1) in iana_subtags + not lang_code or lang_code.group(1) not in iana_subtags ): languages_with_bad_tags.append(lang) return languages_with_bad_tags @@ -342,14 +342,14 @@ def levenshtein_distance(a: str, b: str) -> int: n = len(b) # create two work vectors of integer distances - v1 = [0 for _ in range(0, n + 1)] + v1 = [0 for _ in range(n + 1)] # initialize v0 (the previous row of distances) # this row is A[0][i]: edit distance for an empty s # the distance is just the number of characters to delete from t - v0 = [i for i in range(0, n + 1)] + v0 = list(range(n + 1)) - for i in range(0, m): + for i in range(m): # calculate v1 (current row distances) from the previous row v0 # first element of v1 is A[i+1][0] @@ -357,7 +357,7 @@ def levenshtein_distance(a: str, b: str) -> int: v1[0] = i + 1 # use formula to fill in the rest of the row - for j in range(0, n): + for j in range(n): # calculating costs for A[i+1][j+1] deletion_cost = v0[j + 1] + 1 insertion_cost = v1[j] + 1 @@ -375,7 +375,7 @@ def levenshtein_distance(a: str, b: str) -> int: return v0[n] -def get_expression_lexer() -> re.Scanner: # noqa +def get_expression_lexer() -> re.Scanner: """ Get a expression lexer (scanner) for parsing. """ @@ -439,11 +439,17 @@ def tokenizer(scan, value): lexicon = [(v, get_tokenizer(k)) for k, v in lexer_rules.items()] # re.Scanner is undocumented but has been around since at least 2003 # https://mail.python.org/pipermail/python-dev/2003-April/035075.html - return re.Scanner(lexicon) # noqa + return re.Scanner(lexicon) # Scanner takes a few 100ms to compile so use this shared instance. -ExpLexerToken = namedtuple("ExpLexerToken", ["name", "value", "start", "end"]) +class ExpLexerToken(NamedTuple): + name: str + value: str + start: int + end: int + + EXPRESSION_LEXER = get_expression_lexer() diff --git a/pyxform/validators/enketo_validate/__init__.py b/pyxform/validators/enketo_validate/__init__.py index 931052bc1..ef2935dad 100644 --- a/pyxform/validators/enketo_validate/__init__.py +++ b/pyxform/validators/enketo_validate/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Validate XForms using Enketo validator. """ @@ -24,8 +23,6 @@ class EnketoValidateError(Exception): """Common base class for Enketo validate exceptions.""" - pass - def install_exists(): """ @@ -63,7 +60,7 @@ def check_xform(path_to_xform): :return: warnings or List[str] """ if not install_exists(): - raise EnvironmentError( + raise OSError( "Enketo-validate dependency not found. " "Please use the updater tool to install the latest version." ) @@ -75,14 +72,13 @@ def check_xform(path_to_xform): if timeout: return ["XForm took to long to completely validate."] - else: - if returncode > 0: # Error invalid - raise EnketoValidateError( - "Enketo Validate Errors:\n" + ErrorCleaner.enketo_validate(stderr) - ) - elif returncode == 0: - if stdout: - warnings.append("Enketo Validate Warnings:\n" + stdout) - return warnings - elif returncode < 0: - return ["Bad return code from Enketo Validate."] + elif returncode > 0: # Error invalid + raise EnketoValidateError( + "Enketo Validate Errors:\n" + ErrorCleaner.enketo_validate(stderr) + ) + elif returncode == 0: + if stdout: + warnings.append("Enketo Validate Warnings:\n" + stdout) + return warnings + elif returncode < 0: + return ["Bad return code from Enketo Validate."] diff --git a/pyxform/validators/error_cleaner.py b/pyxform/validators/error_cleaner.py index 4b0080dcd..f837f150b 100644 --- a/pyxform/validators/error_cleaner.py +++ b/pyxform/validators/error_cleaner.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Cleans up error messages from the validators. """ @@ -13,12 +12,9 @@ def _replace_xpath_with_tokens(match): strmatch = match.group() # eliminate e.g /html/body/select1[@ref=/id_string/elId]/item/value # instance('q4')/root/item[...] - if ( - strmatch.startswith("/html/body") - or strmatch.startswith("/root/item") - or strmatch.startswith("/html/head/model/bind") - or strmatch.endswith("/item/value") - ): + if strmatch.startswith( + (("/html/body"), ("/root/item"), ("/html/head/model/bind")) + ) or strmatch.endswith("/item/value"): return strmatch line = match.group().split("/") return "${%s}" % line[len(line) - 1] diff --git a/pyxform/validators/odk_validate/__init__.py b/pyxform/validators/odk_validate/__init__.py index 25dd47968..9616fafb4 100644 --- a/pyxform/validators/odk_validate/__init__.py +++ b/pyxform/validators/odk_validate/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ odk_validate.py A python wrapper around ODK Validate @@ -27,8 +26,6 @@ class ODKValidateError(Exception): """ODK Validation exception error.""" - pass - def install_exists(): """Returns True if ODK_VALIDATE_PATH exists.""" @@ -68,7 +65,7 @@ def check_java_available(): "To fix this, please either: 1) install Java, or 2) run pyxform with the " "--skip_validate flag, or 3) add the installed Java to the environment path." ) - raise EnvironmentError(msg) + raise OSError(msg) def check_xform(path_to_xform): @@ -92,16 +89,15 @@ def check_xform(path_to_xform): if result.timeout: return ["XForm took to long to completely validate."] - else: - if result.return_code > 0: # Error invalid - raise ODKValidateError( - "ODK Validate Errors:\n" + ErrorCleaner.odk_validate(result.stderr) - ) - elif result.return_code == 0: - if result.stderr: - warnings.append("ODK Validate Warnings:\n" + result.stderr) - elif result.return_code < 0: - return ["Bad return code from ODK Validate."] + elif result.return_code > 0: # Error invalid + raise ODKValidateError( + "ODK Validate Errors:\n" + ErrorCleaner.odk_validate(result.stderr) + ) + elif result.return_code == 0: + if result.stderr: + warnings.append("ODK Validate Warnings:\n" + result.stderr) + elif result.return_code < 0: + return ["Bad return code from ODK Validate."] return warnings diff --git a/pyxform/validators/pyxform/select_from_file.py b/pyxform/validators/pyxform/select_from_file.py index 161b76b59..33ace1cce 100644 --- a/pyxform/validators/pyxform/select_from_file.py +++ b/pyxform/validators/pyxform/select_from_file.py @@ -43,7 +43,6 @@ def value_or_label_check(name: str, value: str, row_number: int) -> None: if not value_or_label_test(value=value): msg = value_or_label_format_msg(name=name, row_number=row_number) raise PyXFormError(msg) - return None def validate_list_name_extension( @@ -55,7 +54,7 @@ def validate_list_name_extension( 1 != len(list_path.suffixes) or list_path.suffix not in EXTERNAL_INSTANCE_EXTENSIONS ): - exts = ", ".join((f"'{e}'" for e in EXTERNAL_INSTANCE_EXTENSIONS)) + exts = ", ".join(f"'{e}'" for e in EXTERNAL_INSTANCE_EXTENSIONS) raise PyXFormError( ROW_FORMAT_STRING % row_number + f" File name for '{select_command} {list_name}' should end with one of " diff --git a/pyxform/validators/updater.py b/pyxform/validators/updater.py index 73e5994ba..6cefc099c 100644 --- a/pyxform/validators/updater.py +++ b/pyxform/validators/updater.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- """ pyxform_validator_update - command to update XForm validators. """ import argparse import fnmatch -import io import json import logging import os @@ -64,7 +62,7 @@ def __init__( self.installed_path = os.path.join(self.bin_path, "installed.json") self.bin_new_path = os.path.join(self.mod_path, "bin_new") - self.manual_msg = "Download manually from: {r}.".format(r=self.repo_url) + self.manual_msg = f"Download manually from: {self.repo_url}." class _UpdateHandler: @@ -88,7 +86,7 @@ def _request_latest_json(url): @staticmethod def _check_path(file_path): if not os.path.exists(file_path): - raise PyXFormError("Expected path does not exist: {p}" "".format(p=file_path)) + raise PyXFormError(f"Expected path does not exist: {file_path}" "") else: return True @@ -98,7 +96,7 @@ def _read_json(file_path): Read the JSON file to a string. """ _UpdateHandler._check_path(file_path=file_path) - with io.open(file_path, mode="r") as in_file: + with open(file_path, encoding="utf-8") as in_file: return json.load(in_file) @staticmethod @@ -106,7 +104,7 @@ def _write_json(file_path, content): """ Save the JSON data to a file. """ - with io.open(file_path, mode="w", newline="\n") as out_file: + with open(file_path, mode="w", encoding="utf-8", newline="\n") as out_file: data = json.dumps(content, indent=2, sort_keys=True) out_file.write(str(data)) @@ -116,7 +114,7 @@ def _read_last_check(file_path): Read the .last_check file. """ _UpdateHandler._check_path(file_path=file_path) - with io.open(file_path, mode="r") as in_file: + with open(file_path, encoding="utf-8") as in_file: first_line = in_file.readline() try: last_check = datetime.strptime(first_line, UTC_FMT) @@ -130,7 +128,7 @@ def _write_last_check(file_path, content): """ Write the .last_check file. """ - with io.open(file_path, mode="w", newline="\n") as out_file: + with open(file_path, mode="w", encoding="utf-8", newline="\n") as out_file: out_file.write(str(content.strftime(UTC_FMT))) @staticmethod @@ -173,7 +171,7 @@ def _get_latest(update_info): @staticmethod def _get_release_message(json_data): - template = "- Tag name = {tag_name}\n" "- Tag URL = {tag_url}\n\n" + template = "- Tag name = {tag_name}\n- Tag URL = {tag_url}\n\n" return template.format( tag_name=json_data["tag_name"], tag_url=json_data["html_url"] ) @@ -194,7 +192,7 @@ def list(update_info): latest = _UpdateHandler._get_latest(update_info=update_info) latest_files = latest["assets"] if len(latest_files) == 0: - file_message = "- None!\n\n{m}".format(m=update_info.manual_msg) + file_message = f"- None!\n\n{update_info.manual_msg}" else: file_names = ["- {n}".format(n=x["name"]) for x in latest_files] file_message = "\n".join(file_names) @@ -221,8 +219,7 @@ def _find_download_url(update_info, json_data, file_name): if len(files) == 0: raise PyXFormError( - "No files attached to release '{r}'.\n\n{h}" - "".format(r=rel_name, h=update_info.manual_msg) + f"No files attached to release '{rel_name}'.\n\n{update_info.manual_msg}" ) file_urls = [x["browser_download_url"] for x in files if x["name"] == file_name] @@ -230,8 +227,8 @@ def _find_download_url(update_info, json_data, file_name): urls_len = len(file_urls) if 0 == urls_len: raise PyXFormError( - "No files with the name '{n}' attached to release '{r}'." - "\n\n{h}".format(n=file_name, r=rel_name, h=update_info.manual_msg) + f"No files with the name '{file_name}' attached to release '{rel_name}'." + f"\n\n{update_info.manual_msg}" ) elif 1 < urls_len: raise PyXFormError( @@ -248,7 +245,7 @@ def _download_file(url, file_path): """ Save response content from the URL to a binary file at the file path. """ - with io.open(file_path, mode="wb") as out_file: + with open(file_path, mode="wb") as out_file: file_data = request_get(url=url) out_file.write(file_data) @@ -270,8 +267,9 @@ def _get_bin_paths(update_info, file_path): main_bin = "*validate" else: raise PyXFormError( - "Did not find a supported main binary for file: {p}.\n\n{h}" - "".format(p=file_path, h=update_info.manual_msg) + "Did not find a supported main binary for file: {p}.\n\n{h}".format( + p=file_path, h=update_info.manual_msg + ) ) return [ (main_bin, update_info.validator_basename), @@ -309,8 +307,7 @@ def _unzip_find_jobs(open_zip_file, bin_paths, out_path): zip_jobs[file_out_path] = zip_item if len(bin_paths) != len(zip_jobs.keys()): raise PyXFormError( - "Expected {e} zip job files, found: {c}" - "".format(e=len(bin_paths), c=len(zip_jobs.keys())) + f"Expected {len(bin_paths)} zip job files, found: {len(zip_jobs.keys())}" ) return zip_jobs @@ -328,7 +325,7 @@ def _unzip_extract_file(open_zip_file, zip_item, file_out_path): os.makedirs(out_parent) with open_zip_file.open(zip_item, mode="r") as zip_item_file: zip_item_data = zip_item_file.read() - with io.open(file_out_path, "wb") as file_out_file: + with open(file_out_path, "wb") as file_out_file: file_out_file.write(zip_item_data) @staticmethod @@ -389,8 +386,8 @@ def _install(update_info, file_name): current_mode = os.stat(new_bin_file_path).st_mode os.chmod(new_bin_file_path, current_mode | S_IXUSR | S_IXGRP) - except PyXFormError as e: - raise PyXFormError("\n\nUpdate failed!\n\n" + str(e)) + except PyXFormError as px_err: + raise PyXFormError("\n\nUpdate failed!\n\n" + str(px_err)) from px_err else: return latest @@ -497,7 +494,6 @@ def check(update_info): class _UpdateService: - update_info = None def list(self): @@ -556,11 +552,10 @@ def _install_check(bin_file_path=None): def _build_validator_menu(main_subparser, validator_name, updater_instance): - main = main_subparser.add_parser( validator_name.lower(), - description="{v} Sub-menu".format(v=validator_name), - help="For help, use '{v} -h'.".format(v=validator_name.lower()), + description=f"{validator_name} Sub-menu", + help=f"For help, use '{validator_name.lower()} -h'.", ) subs = main.add_subparsers(metavar="") diff --git a/pyxform/validators/util.py b/pyxform/validators/util.py index 880e55d32..ce09682c4 100644 --- a/pyxform/validators/util.py +++ b/pyxform/validators/util.py @@ -1,9 +1,6 @@ -# -*- coding: utf-8 -*- """ The validators utility functions. """ -import collections -import io import logging import os import signal @@ -13,6 +10,7 @@ import time from contextlib import closing from subprocess import PIPE, Popen +from typing import Dict, List, NamedTuple from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -48,7 +46,6 @@ def _kill_process_after_a_timeout(pid): os.kill(pid, signal.SIGTERM) kill_check.set() # tell the main routine that we had to kill # use SIGKILL if hard to kill... - return startup_info = None env = None @@ -101,7 +98,7 @@ def decode_stream(stream): return stream.decode("latin-1") except BaseException as be: msg = "Failed to decode validate stderr as utf-8 or latin-1." - raise IOError(msg, ude, be) + raise OSError(msg, ude, be) from be def request_get(url): @@ -109,24 +106,31 @@ def request_get(url): Get the response content from URL. """ try: + if not url.startswith(("http:", "https:")): + raise ValueError("URL must start with 'http:' or 'https:'") r = Request(url) r.add_header("Accept", "application/json") with closing(urlopen(r)) as u: content = u.read() if len(content) == 0: - raise PyXFormError("Empty response from URL: '{u}'.".format(u=url)) + raise PyXFormError(f"Empty response from URL: '{url}'.") else: return content - except HTTPError as e: + except HTTPError as http_err: raise PyXFormError( - "Unable to fulfill request. Error code: '{c}'. " - "Reason: '{r}'. URL: '{u}'." - "".format(r=e.reason, c=e.code, u=url) - ) - except URLError as e: + f"Unable to fulfill request. Error code: '{http_err.code}'. " + f"Reason: '{http_err.reason}'. URL: '{url}'." + "" + ) from http_err + except URLError as url_err: raise PyXFormError( - "Unable to reach a server. Reason: {r}. " "URL: {u}".format(r=e.reason, u=url) - ) + f"Unable to reach a server. Reason: {url_err.reason}. " f"URL: {url}" + ) from url_err + + +class _LoggingWatcher(NamedTuple): + records: List + output: Dict class CapturingHandler(logging.Handler): @@ -166,7 +170,6 @@ def emit(self, record): @staticmethod def _get_watcher(): - _LoggingWatcher = collections.namedtuple("_LoggingWatcher", ["records", "output"]) levels = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"] return _LoggingWatcher([], {x: [] for x in levels}) @@ -190,9 +193,9 @@ def check_readable(file_path, retry_limit=10, wait_seconds=0.5): def catch_try(): try: - with io.open(file_path, mode="r"): + with open(file_path): return True - except IOError: + except OSError: return False tries = 0 @@ -201,5 +204,5 @@ def catch_try(): tries += 1 time.sleep(wait_seconds) else: - raise IOError("Could not read file: {f}".format(f=file_path)) + raise OSError(f"Could not read file: {file_path}") return True diff --git a/pyxform/xform2json.py b/pyxform/xform2json.py index 4cf0ebccf..e6d458b83 100644 --- a/pyxform/xform2json.py +++ b/pyxform/xform2json.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- """ xform2json module - Transform an XForm to a JSON dictionary. """ -import codecs import copy import json import logging import re -import xml.etree.ElementTree as ETree -from collections import Mapping +from collections.abc import Mapping from operator import itemgetter from typing import Any, Dict, List +from xml.etree.ElementTree import Element + +from defusedxml.ElementTree import ParseError, XMLParser, fromstring, parse from pyxform import builder from pyxform.errors import PyXFormError @@ -20,6 +20,15 @@ logger.addHandler(logging.NullHandler()) +QUESTION_TYPES = { + "select": "select all that apply", + "select1": "select one", + "int": "integer", + "dateTime": "datetime", + "string": "text", +} + + # {{{ http://code.activestate.com/recipes/573463/ (r7) class XmlDictObject(dict): """ @@ -59,7 +68,7 @@ def wrap(x): @staticmethod def _un_wrap(x): if isinstance(x, dict): - return dict((k, XmlDictObject._un_wrap(v)) for (k, v) in iter(x.items())) + return {k: XmlDictObject._un_wrap(v) for k, v in x.items()} elif isinstance(x, list): return [XmlDictObject._un_wrap(v) for v in x] else: @@ -75,20 +84,21 @@ def un_wrap(self): def _convert_dict_to_xml_recurse(parent, dictitem): - assert not isinstance(dictitem, list) + if isinstance(dictitem, list): + raise PyXFormError("""Invalid value for `dictitem`.""") if isinstance(dictitem, dict): - for (tag, child) in iter(dictitem.items()): + for tag, child in iter(dictitem.items()): if str(tag) == "_text": parent.text = str(child) elif isinstance(child, list): # iterate through the array and convert for listchild in child: - elem = ETree.Element(tag) + elem = Element(tag) parent.append(elem) _convert_dict_to_xml_recurse(elem, listchild) else: - elem = ETree.Element(tag) + elem = Element(tag) parent.append(elem) _convert_dict_to_xml_recurse(elem, child) else: @@ -101,7 +111,7 @@ def convert_dict_to_xml(xmldict): """ roottag = xmldict.keys()[0] - root = ETree.Element(roottag) + root = Element(roottag) _convert_dict_to_xml_recurse(root, xmldict[roottag]) return root @@ -155,7 +165,7 @@ def convert_xml_to_dict(root, dictclass=XmlDictObject): # If a string is passed in, try to open it as a file if isinstance(root, str): root = _try_parse(root) - elif not isinstance(root, ETree.Element): + elif not isinstance(root, Element): raise TypeError("Expected ElementTree.Element or file path string") return dictclass({root.tag: _convert_xml_to_dict_recurse(root, dictclass)}) @@ -170,21 +180,21 @@ def _try_parse(root, parser=None): """ root = root.encode("UTF-8") try: - parsed_root = ETree.fromstring(root, parser) - except ETree.ParseError: - parsed_root = ETree.parse(root, parser=parser).getroot() + parsed_root = fromstring(root, parser) + except ParseError: + parsed_root = parse(root, parser=parser).getroot() return parsed_root class XFormToDict: def __init__(self, root): if isinstance(root, str): - parser = ETree.XMLParser(encoding="UTF-8") + parser = XMLParser(encoding="UTF-8") self._root = _try_parse(root, parser) self._dict = XmlDictObject( {self._root.tag: _convert_xml_to_dict_recurse(self._root, XmlDictObject)} ) - elif not isinstance(root, ETree.Element): + elif not isinstance(root, Element): raise TypeError("Expected ElementTree.Element or file path string") def get_dict(self): @@ -202,24 +212,22 @@ def create_survey_element_from_xml(xml_file): class XFormToDictBuilder: """Experimental XFORM xml to XFORM JSON""" - QUESTION_TYPES = { - "select": "select all that apply", - "select1": "select one", - "int": "integer", - "dateTime": "datetime", - "string": "text", - } - def __init__(self, xml_file): doc_as_dict = XFormToDict(xml_file).get_dict() self._xmldict = doc_as_dict - assert "html" in doc_as_dict - assert "body" in doc_as_dict["html"] - assert "head" in doc_as_dict["html"] - assert "model" in doc_as_dict["html"]["head"] - assert "title" in doc_as_dict["html"]["head"] - assert "bind" in doc_as_dict["html"]["head"]["model"] + if "html" not in doc_as_dict: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "body" not in doc_as_dict["html"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "head" not in doc_as_dict["html"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "model" not in doc_as_dict["html"]["head"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "title" not in doc_as_dict["html"]["head"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") + if "bind" not in doc_as_dict["html"]["head"]["model"]: + raise PyXFormError("""Invalid value for `doc_as_dict`.""") self.body = doc_as_dict["html"]["body"] self.model = doc_as_dict["html"]["head"]["model"] @@ -357,7 +365,7 @@ def _get_item_func(self, ref, name, item): rs = {} name_splits = name.split("/") rs["name"] = name_splits[0] - ref = "%s/%s" % (ref, rs["name"]) + ref = f"""{ref}/{rs["name"]}""" rs["ref"] = ref if name_splits.__len__() > 1: rs["type"] = "group" @@ -385,8 +393,10 @@ def _get_question_from_object(self, obj, type=None): except KeyError: try: ref = obj["nodeset"] - except KeyError: - raise TypeError('cannot find "ref" or "nodeset" in {}'.format(repr(obj))) + except KeyError as node_err: + raise PyXFormError( + f'Cannot find "ref" or "nodeset" in {obj!r}' + ) from node_err question = { "ref": ref, "__order": self._get_question_order(ref), @@ -459,7 +469,7 @@ def _get_question_from_object(self, obj, type=None): "select1", "select", ]: # Select bind type is 'string' https://github.com/XLSForm/pyxform/issues/168 - question["type"] = self.QUESTION_TYPES[type] + question["type"] = QUESTION_TYPES[type] if question_type == "geopoint" and "hint" in question: del question["hint"] if "type" not in question and type: @@ -505,7 +515,7 @@ def _get_question_params_from_bindings(self, ref): if k == "nodeset": continue if k == "type": - v = self._get_question_type(v) + v = self._get_question_type(question_type=v) if k in [ "relevant", "required", @@ -546,20 +556,24 @@ def _get_question_params_from_bindings(self, ref): return rs return None - def _get_question_type(self, type): - if type in self.QUESTION_TYPES.keys(): - return self.QUESTION_TYPES[type] - return type + @staticmethod + def _get_question_type(question_type): + if question_type in QUESTION_TYPES.keys(): + return QUESTION_TYPES[question_type] + return question_type def _get_translations(self) -> List[Dict]: if "itext" not in self.model: return [] - assert "translation" in self.model["itext"] + if "translation" not in self.model["itext"]: + raise PyXFormError("""Invalid value for `self.model["itext"]`.""") translations = self.model["itext"]["translation"] if isinstance(translations, dict): translations = [translations] - assert "text" in translations[0] - assert "lang" in translations[0] + if "text" not in translations[0]: + raise PyXFormError("""Invalid value for `translations[0]`.""") + if "lang" not in translations[0]: + raise PyXFormError("""Invalid value for `translations[0]`.""") return translations def _get_label(self, label_obj, key="label"): @@ -702,6 +716,6 @@ def replace_function(match): def write_object_to_file(filename, obj): """Writes to a JSON filename the dictionary obj.""" - with codecs.open(filename, "w", encoding="utf-8") as output_file: + with open(filename, "w", encoding="utf-8") as output_file: output_file.write(json.dumps(obj, indent=2)) logger.info("object written to file: %s", filename) diff --git a/pyxform/xform_instance_parser.py b/pyxform/xform_instance_parser.py index d6e31e2c6..7c41dc1de 100644 --- a/pyxform/xform_instance_parser.py +++ b/pyxform/xform_instance_parser.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ XFormInstanceParser class module - parses an instance XML. """ @@ -6,13 +5,18 @@ # where this code is actually going to live. import re -from xml.dom import minidom +from xml.dom.minidom import Node + +from defusedxml.minidom import parseString + +from pyxform.errors import PyXFormError XFORM_ID_STRING = "_xform_id_string" def _xml_node_to_dict(node): - assert isinstance(node, minidom.Node) + if not isinstance(node, Node): + raise PyXFormError("""Invalid value for `node`.""") if len(node.childNodes) == 0: # there's no data for this leaf node value = None @@ -25,11 +29,12 @@ def _xml_node_to_dict(node): for child in node.childNodes: d = _xml_node_to_dict(child) child_name = child.nodeName - assert list(d.keys()) == [child_name] + if list(d.keys()) != [child_name]: + raise PyXFormError("""Invalid value for `d`.""") if child_name not in value: # copy the value into the dict value[child_name] = d[child_name] - elif type(value[child_name]) == list: + elif isinstance(value[child_name], list): # add to the existing list value[child_name].append(d[child_name]) else: @@ -42,15 +47,17 @@ def _flatten_dict(d, prefix): """ Return a list of XPath, value pairs. """ - assert type(d) == dict - assert type(prefix) == list + if not isinstance(d, dict): + raise PyXFormError("""Invalid value for `d`.""") + if not isinstance(prefix, list): + raise PyXFormError("""Invalid value for `prefix`.""") for key, value in d.items(): - new_prefix = prefix + [key] - if type(value) == dict: + new_prefix = [*prefix, key] + if isinstance(value, dict): for pair in _flatten_dict(value, new_prefix): yield pair - elif type(value) == list: + elif isinstance(value, list): for i, item in enumerate(value): item_prefix = list(new_prefix) # make a copy # note on indexing xpaths: IE5 and later has @@ -58,7 +65,7 @@ def _flatten_dict(d, prefix): # according to the W3C standard it should have been # [1]. I'm adding 1 to i to start at 1. item_prefix[-1] += "[%s]" % str(i + 1) - if type(item) == dict: + if isinstance(item, dict): for pair in _flatten_dict(item, item_prefix): yield pair else: @@ -75,8 +82,7 @@ def _get_all_attributes(node): for key in node.attributes.keys(): yield key, node.getAttribute(key) for child in node.childNodes: - for pair in _get_all_attributes(child): - yield pair + yield from _get_all_attributes(child) class XFormInstanceParser: @@ -85,8 +91,8 @@ def __init__(self, xml_str): def parse(self, xml_str): clean_xml_str = xml_str.strip() - clean_xml_str = re.sub(str(r">\s+<"), str("><"), clean_xml_str) - self._xml_obj = minidom.parseString(clean_xml_str) + clean_xml_str = re.sub(r">\s+<", "><", clean_xml_str) + self._xml_obj = parseString(clean_xml_str) self._root_node = self._xml_obj.documentElement self._dict = _xml_node_to_dict(self._root_node) self._flat_dict = {} @@ -113,7 +119,8 @@ def _set_attributes(self): self._attributes = {} all_attributes = list(_get_all_attributes(self._root_node)) for key, value in all_attributes: - assert key not in self._attributes + if key in self._attributes: + raise PyXFormError("""Invalid value for `self._attributes`.""") self._attributes[key] = value def get_xform_id_string(self): diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index 02fe8aa2f..3a019fb14 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- """ A Python script to convert excel files into JSON. """ -import codecs import json import os import re @@ -39,9 +37,8 @@ def print_pyobj_to_json(pyobj, path=None): or stdout if no file is specified """ if path: - fp = codecs.open(path, mode="w", encoding="utf-8") - json.dump(pyobj, fp=fp, ensure_ascii=False, indent=4) - fp.close() + with open(path, mode="w", encoding="utf-8") as fp: + json.dump(pyobj, fp=fp, ensure_ascii=False, indent=4) else: sys.stdout.write(json.dumps(pyobj, ensure_ascii=False, indent=4)) @@ -72,7 +69,7 @@ def merge_dicts(dict_a, dict_b, default_key="default"): all_keys = {k: None for k in dict_a.keys()} all_keys.update({k: None for k in dict_b.keys()}) - out_dict = dict() + out_dict = {} for key in all_keys.keys(): out_dict[key] = merge_dicts(dict_a.get(key), dict_b.get(key), default_key) return out_dict @@ -131,10 +128,10 @@ def dealias_and_group_headers( without a language specified with localized versions. """ group_delimiter = "::" - out_dict_array = list() + out_dict_array = [] seen_headers = {} for row in dict_array: - out_row = dict() + out_row = {} for header, val in row.items(): if ignore_case: header = header.lower() @@ -230,7 +227,7 @@ def group_dictionaries_by_key(list_of_dicts, key, remove_key=True): The grouping key is removed by default. If the key is not in any dictionary an empty dict is returned. """ - dict_of_lists = dict() + dict_of_lists = {} for dicty in list_of_dicts: if key not in dicty: continue @@ -252,7 +249,7 @@ def has_double_colon(workbook_dict) -> bool: for sheet in workbook_dict.values(): for row in sheet: for column_header in row.keys(): - if type(column_header) is not str: + if not isinstance(column_header, str): continue if "::" in column_header: return True @@ -287,12 +284,11 @@ def add_flat_annotations(prompt_list, parent_relevant="", name_prefix=""): add_flat_annotations( children, new_relevant, name_prefix + "_" + prompt["name"] ) - else: - if new_relevant != "": - prompt["bind"] = prompt.get("bind", {}) - prompt["bind"]["relevant"] = new_relevant - # if name_prefix != '': - # prompt['name'] = name_prefix + prompt['name'] + elif new_relevant != "": + prompt["bind"] = prompt.get("bind", {}) + prompt["bind"]["relevant"] = new_relevant + # if name_prefix != '': + # prompt['name'] = name_prefix + prompt['name'] def process_range_question_type( @@ -315,12 +311,16 @@ def process_range_question_type( if key not in parameters: parameters[key] = defaults[key] + has_float = False try: - has_float = any([float(x) and "." in str(x) for x in parameters.values()]) - except ValueError: + # Check all parameters. + for x in parameters.values(): + if float(x) and "." in str(x): + has_float = True + except ValueError as range_err: raise PyXFormError( - "Range parameters 'start', " "'end' or 'step' must all be numbers." - ) + "Range parameters 'start', 'end' or 'step' must all be numbers." + ) from range_err else: # is integer by default, convert to decimal if it has any float values if has_float: @@ -422,7 +422,7 @@ def workbook_to_json( workbook_dict = {x.lower(): y for x, y in workbook_dict.items()} workbook_keys = workbook_dict.keys() if constants.SURVEY not in workbook_dict: - msg = "You must have a sheet named '{k}'. ".format(k=constants.SURVEY) + msg = f"You must have a sheet named '{constants.SURVEY}'. " similar = find_sheet_misspellings(key=constants.SURVEY, keys=workbook_keys) if similar is not None: msg += similar @@ -556,7 +556,7 @@ def workbook_to_json( if "name" not in option: info = "[list_name : " + list_name + "]" raise PyXFormError( - "On the choices sheet there is " "a option with no name. " + info + "On the choices sheet there is a option with no name. " + info ) if "label" not in option: info = "[list_name : " + list_name + "]" @@ -564,7 +564,7 @@ def workbook_to_json( "On the choices sheet there is a option with no label. " + info ) # chrislrobert's fix for a cryptic error message: - # see: https://code.google.com/p/opendatakit/issues/detail?id=832&start=200 # noqa + # see: https://code.google.com/p/opendatakit/issues/detail?id=833&start=200 option_keys = list(option.keys()) for headername in option_keys: # Using warnings and removing the bad columns @@ -590,7 +590,7 @@ def workbook_to_json( list_name_choices = [option.get("name") for option in options] if len(list_name_choices) != len(set(list_name_choices)): duplicate_setting = settings.get("allow_choice_duplicates") - for k, v in Counter(list_name_choices).items(): + for v in Counter(list_name_choices).values(): if v > 1: if not duplicate_setting or duplicate_setting.capitalize() != "Yes": choice_duplicates = [ @@ -607,13 +607,10 @@ def workbook_to_json( "allow_choice_duplicates setting to 'yes'. Learn more: https://xlsform.org/#choice-names.".format( list_name, ", ".join( - [ - "'{}'".format(dupe) - for dupe in choice_duplicates - ] + [f"'{dupe}'" for dupe in choice_duplicates] ), ) - ) # noqa + ) # ########## Entities sheet ########### entities_sheet = workbook_dict.get(constants.ENTITIES, []) @@ -782,8 +779,9 @@ def workbook_to_json( new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update( { - "odk:" - + constants.TRACK_CHANGES: parameters[constants.TRACK_CHANGES] + "odk:" + constants.TRACK_CHANGES: parameters[ + constants.TRACK_CHANGES + ] } ) @@ -811,8 +809,9 @@ def workbook_to_json( new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update( { - "odk:" - + constants.IDENTIFY_USER: parameters[constants.IDENTIFY_USER] + "odk:" + constants.IDENTIFY_USER: parameters[ + constants.IDENTIFY_USER + ] } ) @@ -823,7 +822,6 @@ def workbook_to_json( ) if any(k in parameters.keys() for k in location_parameters): if all(k in parameters.keys() for k in location_parameters): - if parameters[constants.LOCATION_PRIORITY] not in [ "no-power", "low-power", @@ -840,12 +838,12 @@ def workbook_to_json( try: int(parameters[constants.LOCATION_MIN_INTERVAL]) - except ValueError: + except ValueError as lmi_err: raise PyXFormError( "Parameter " + constants.LOCATION_MIN_INTERVAL + " must have an integer value." - ) + ) from lmi_err if int(parameters[constants.LOCATION_MIN_INTERVAL]) < 0: raise PyXFormError( "Parameter " @@ -855,12 +853,12 @@ def workbook_to_json( try: int(parameters[constants.LOCATION_MAX_AGE]) - except ValueError: + except ValueError as lma_err: raise PyXFormError( "Parameter " + constants.LOCATION_MAX_AGE + " must have an integer value." - ) + ) from lma_err if int(parameters[constants.LOCATION_MAX_AGE]) < 0: raise PyXFormError( "Parameter " @@ -882,16 +880,13 @@ def workbook_to_json( new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update( { - "odk:" - + constants.LOCATION_MAX_AGE: parameters[ + "odk:" + constants.LOCATION_MAX_AGE: parameters[ constants.LOCATION_MAX_AGE ], - "odk:" - + constants.LOCATION_MIN_INTERVAL: parameters[ + "odk:" + constants.LOCATION_MIN_INTERVAL: parameters[ constants.LOCATION_MIN_INTERVAL ], - "odk:" - + constants.LOCATION_PRIORITY: parameters[ + "odk:" + constants.LOCATION_PRIORITY: parameters[ constants.LOCATION_PRIORITY ], } @@ -1027,7 +1022,7 @@ def workbook_to_json( new_json_dict = row.copy() new_json_dict[constants.TYPE] = control_type - child_list = list() + child_list = [] new_json_dict[constants.CHILDREN] = child_list if control_type is constants.LOOP: if not parse_dict.get("list_name"): @@ -1127,7 +1122,8 @@ def workbook_to_json( and constants.CHOICE_FILTER not in row ): warnings.append( - ROW_FORMAT_STRING % row_number + ROW_FORMAT_STRING + % row_number + " select one external is only meant for" " filtered selects." ) @@ -1237,10 +1233,10 @@ def workbook_to_json( if not parameters["seed"].startswith("${"): try: float(parameters["seed"]) - except ValueError: + except ValueError as seed_err: raise PyXFormError( "seed value must be a number or a reference to another field." - ) + ) from seed_err elif "seed" in parameters.keys(): raise PyXFormError( "Parameters must include randomize=true to use a seed." @@ -1295,7 +1291,7 @@ def workbook_to_json( constants.TYPE: select_type, constants.NAME: "reserved_name_for_field_list_labels_" + str(row_number), - # Adding row number for uniqueness # noqa + # Adding row number for uniqueness constants.CONTROL: {constants.APPEARANCE: "label"}, constants.CHOICES: choices[list_name], constants.ITEMSET: list_name, @@ -1351,11 +1347,11 @@ def workbook_to_json( if "rows" in parameters.keys(): try: int(parameters["rows"]) - except ValueError: + except ValueError as rows_err: raise PyXFormError( (ROW_FORMAT_STRING % row_number) + " Parameter rows must have an integer value." - ) + ) from rows_err new_dict["control"] = new_dict.get("control", {}) new_dict["control"].update({"rows": parameters["rows"]}) @@ -1378,8 +1374,10 @@ def workbook_to_json( if "max-pixels" in parameters.keys(): try: int(parameters["max-pixels"]) - except ValueError: - raise PyXFormError("Parameter max-pixels must have an integer value.") + except ValueError as mp_err: + raise PyXFormError( + "Parameter max-pixels must have an integer value." + ) from mp_err new_dict["bind"] = new_dict.get("bind", {}) new_dict["bind"].update({"orx:max-pixels": parameters["max-pixels"]}) @@ -1475,10 +1473,10 @@ def workbook_to_json( new_dict["control"].update( {"accuracyThreshold": parameters["capture-accuracy"]} ) - except ValueError: + except ValueError as ca_err: raise PyXFormError( "Parameter capture-accuracy must have a numeric value" - ) + ) from ca_err if "warning-accuracy" in parameters.keys(): try: @@ -1486,10 +1484,10 @@ def workbook_to_json( new_dict["control"].update( {"unacceptableAccuracyThreshold": parameters["warning-accuracy"]} ) - except ValueError: + except ValueError as wa_err: raise PyXFormError( "Parameter warning-accuracy must have a numeric value" - ) + ) from wa_err parent_children_array.append(new_dict) continue @@ -1510,7 +1508,7 @@ def workbook_to_json( # print "Generating flattened instance..." add_flat_annotations(stack[0]["parent_children"]) - meta_children = [] + survey_meta + meta_children = [*survey_meta] if aliases.yes_no.get(settings.get("omit_instanceID")): if settings.get("public_key"): @@ -1589,7 +1587,7 @@ def get_filename(path): """ Get the extensionless filename from a path """ - return os.path.splitext((os.path.basename(path)))[0] + return os.path.splitext(os.path.basename(path))[0] def parse_file_to_json( @@ -1630,7 +1628,7 @@ def organize_by_values(dict_list, key): dicty_copy = dicty.copy() val = dicty_copy.pop(key) if val in result: - raise Exception("Duplicate key: " + val) + raise PyXFormError("Duplicate key: " + val) result[val] = dicty_copy return result @@ -1683,7 +1681,7 @@ def __init__(self, path_or_file, default_name=None): def print_warning_log(self, warn_out_file): # Open file to print warning log to. - warn_out = open(warn_out_file, "w") + warn_out = open(warn_out_file, mode="w", encoding="utf-8") warn_out.write("\n".join(self._warnings)) @@ -1695,7 +1693,7 @@ class QuestionTypesReader(SpreadsheetReader): """ def __init__(self, path): - super(QuestionTypesReader, self).__init__(path) + super().__init__(path) self._setup_question_types_dictionary() def _setup_question_types_dictionary(self): @@ -1711,28 +1709,6 @@ def _setup_question_types_dictionary(self): self._dict = organize_by_values(self._dict, "name") -# Not used internally (or on formhub) -# TODO: If this is used anywhere else it is probably broken -# from the changes I made to SpreadsheetReader. -class VariableNameReader(SpreadsheetReader): - def __init__(self, path): - SpreadsheetReader.__init__(self, path) - self._organize_renames() - - def _organize_renames(self): - new_dict = {} - variable_names_so_far = [] - assert "Dictionary" in self._dict - for d in self._dict["Dictionary"]: - if "Variable Name" in d: - assert d["Variable Name"] not in variable_names_so_far, d["Variable Name"] - variable_names_so_far.append(d["Variable Name"]) - new_dict[d["XPath"]] = d["Variable Name"] - else: - variable_names_so_far.append(d["XPath"]) - self._dict = new_dict - - if __name__ == "__main__": # Open the excel file specified by the argument of this python call, # convert that file to json, then print it diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 93155edda..eca89014b 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -1,20 +1,25 @@ -# -*- coding: utf-8 -*- """ XLS-to-dict and csv-to-dict are essentially backends for xls2json. """ import csv import datetime +import os import re from collections import OrderedDict +from contextlib import closing from functools import reduce from io import StringIO from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union from zipfile import BadZipFile -import openpyxl import xlrd from openpyxl.cell import Cell as pyxlCell +from openpyxl.reader.excel import ExcelReader +from openpyxl.workbook import Workbook as pyxlWorkbook +from openpyxl.worksheet.worksheet import Worksheet as pyxlWorksheet +from xlrd.book import Book as xlrdBook from xlrd.sheet import Cell as xlrdCell +from xlrd.sheet import Sheet as xlrdSheet from xlrd.xldate import XLDateAmbiguous from pyxform import constants @@ -53,7 +58,7 @@ def trim_trailing_empty(a_list: list, n_empty: int) -> list: def get_excel_column_headers(first_row: Iterator[Optional[str]]) -> List[Optional[str]]: """Get column headers from the first row; stop if there's a run of empty columns.""" max_adjacent_empty_columns = 20 - column_header_list = list() + column_header_list = [] adjacent_empty_cols = 0 for column_header in first_row: if is_empty(column_header): @@ -121,59 +126,76 @@ def xls_to_dict(path_or_file): equal to the cell value for that row and column. All the keys and leaf elements are unicode text. """ - try: - if isinstance(path_or_file, str): - workbook = xlrd.open_workbook(filename=path_or_file) - else: - workbook = xlrd.open_workbook(file_contents=path_or_file.read()) - except xlrd.XLRDError as error: - raise PyXFormError("Error reading .xls file: %s" % error) - def xls_clean_cell(cell: xlrdCell, row_n: int, col_key: str) -> Optional[str]: + def xls_clean_cell( + wb: xlrdBook, wb_sheet: xlrdSheet, cell: xlrdCell, row_n: int, col_key: str + ) -> Optional[str]: value = cell.value if isinstance(value, str): value = value.strip() if not is_empty(value): try: - return xls_value_to_unicode(value, cell.ctype, workbook.datemode) - except XLDateAmbiguous: - raise PyXFormError(XL_DATE_AMBIGOUS_MSG % (wb_sheet.name, col_key, row_n)) + return xls_value_to_unicode(value, cell.ctype, wb.datemode) + except XLDateAmbiguous as date_err: + raise PyXFormError( + XL_DATE_AMBIGOUS_MSG % (wb_sheet.name, col_key, row_n) + ) from date_err return None - def xls_to_dict_normal_sheet(sheet): + def xls_to_dict_normal_sheet(wb: xlrdBook, wb_sheet: xlrdSheet): # XLS format: max cols 256, max rows 65536 - first_row = (c.value for c in next(sheet.get_rows(), [])) + first_row = (c.value for c in next(wb_sheet.get_rows(), [])) headers = get_excel_column_headers(first_row=first_row) row_iter = ( - tuple(sheet.cell(r, c) for c in range(0, len(headers))) - for r in range(1, sheet.nrows) + tuple(wb_sheet.cell(r, c) for c in range(len(headers))) + for r in range(1, wb_sheet.nrows) ) - rows = get_excel_rows(headers=headers, rows=row_iter, cell_func=xls_clean_cell) + + # Inject wb/sheet as closure since functools.partial isn't typing friendly. + def clean_func(cell: xlrdCell, row_n: int, col_key: str) -> Optional[str]: + return xls_clean_cell( + wb=wb, wb_sheet=wb_sheet, cell=cell, row_n=row_n, col_key=col_key + ) + + rows = get_excel_rows(headers=headers, rows=row_iter, cell_func=clean_func) column_header_list = [key for key in headers if key is not None] return rows, _list_to_dict_list(column_header_list) - result_book = OrderedDict() - for wb_sheet in workbook.sheets(): - # Note that the sheet exists but do no further processing here. - result_book[wb_sheet.name] = [] - # Do not process sheets that have nothing to do with XLSForm. - if wb_sheet.name not in constants.SUPPORTED_SHEET_NAMES: - if len(workbook.sheets()) == 1: - ( - result_book[constants.SURVEY], - result_book["%s_header" % constants.SURVEY], - ) = xls_to_dict_normal_sheet(wb_sheet) + def process_workbook(wb: xlrdBook): + result_book = OrderedDict() + for wb_sheet in wb.sheets(): + # Note that the sheet exists but do no further processing here. + result_book[wb_sheet.name] = [] + # Do not process sheets that have nothing to do with XLSForm. + if wb_sheet.name not in constants.SUPPORTED_SHEET_NAMES: + if len(wb.sheets()) == 1: + ( + result_book[constants.SURVEY], + result_book["%s_header" % constants.SURVEY], + ) = xls_to_dict_normal_sheet(wb=wb, wb_sheet=wb_sheet) + else: + continue else: - continue - else: - ( - result_book[wb_sheet.name], - result_book["%s_header" % wb_sheet.name], - ) = xls_to_dict_normal_sheet(wb_sheet) + ( + result_book[wb_sheet.name], + result_book["%s_header" % wb_sheet.name], + ) = xls_to_dict_normal_sheet(wb=wb, wb_sheet=wb_sheet) + return result_book - workbook.release_resources() - return result_book + try: + if isinstance(path_or_file, (str, bytes, os.PathLike)): + file = open(path_or_file, mode="rb") + else: + file = path_or_file + with closing(file) as wb_file: + workbook = xlrd.open_workbook(file_contents=wb_file.read()) + try: + return process_workbook(wb=workbook) + finally: + workbook.release_resources() + except xlrd.XLRDError as read_err: + raise PyXFormError("Error reading .xls file: %s" % read_err) from read_err def xls_value_to_unicode(value, value_type, datemode) -> str: @@ -215,10 +237,6 @@ def xlsx_to_dict(path_or_file): equal to the cell value for that row and column. All the keys and leaf elements are strings. """ - try: - workbook = openpyxl.open(filename=path_or_file, read_only=True, data_only=True) - except (OSError, BadZipFile, KeyError) as error: - raise PyXFormError("Error reading .xlsx file: %s" % error) def xlsx_clean_cell(cell: pyxlCell, row_n: int, col_key: str) -> Optional[str]: value = cell.value @@ -229,7 +247,7 @@ def xlsx_clean_cell(cell: pyxlCell, row_n: int, col_key: str) -> Optional[str]: return None - def xlsx_to_dict_normal_sheet(sheet): + def xlsx_to_dict_normal_sheet(sheet: pyxlWorksheet): # XLSX format: max cols 16384, max rows 1048576 first_row = (c.value for c in next(sheet.rows, [])) headers = get_excel_column_headers(first_row=first_row) @@ -238,28 +256,43 @@ def xlsx_to_dict_normal_sheet(sheet): column_header_list = [key for key in headers if key is not None] return rows, _list_to_dict_list(column_header_list) - result_book = OrderedDict() - for sheetname in workbook.sheetnames: - wb_sheet = workbook[sheetname] - # Note that the sheet exists but do no further processing here. - result_book[sheetname] = [] - # Do not process sheets that have nothing to do with XLSForm. - if sheetname not in constants.SUPPORTED_SHEET_NAMES: - if len(workbook.sheetnames) == 1: + def process_workbook(wb: pyxlWorkbook): + result_book = OrderedDict() + for sheetname in wb.sheetnames: + wb_sheet = wb[sheetname] + # Note that the sheet exists but do no further processing here. + result_book[sheetname] = [] + # Do not process sheets that have nothing to do with XLSForm. + if sheetname not in constants.SUPPORTED_SHEET_NAMES: + if len(wb.sheetnames) == 1: + ( + result_book[constants.SURVEY], + result_book[f"{constants.SURVEY}_header"], + ) = xlsx_to_dict_normal_sheet(wb_sheet) + else: + continue + else: ( - result_book[constants.SURVEY], - result_book[f"{constants.SURVEY}_header"], + result_book[sheetname], + result_book[f"{sheetname}_header"], ) = xlsx_to_dict_normal_sheet(wb_sheet) - else: - continue - else: - ( - result_book[sheetname], - result_book[f"{sheetname}_header"], - ) = xlsx_to_dict_normal_sheet(wb_sheet) + return result_book - workbook.close() - return result_book + try: + if isinstance(path_or_file, (str, bytes, os.PathLike)): + file = open(path_or_file, mode="rb") + else: + file = path_or_file + with closing(file) as wb_file: + reader = ExcelReader(wb_file, read_only=True, data_only=True) + reader.read() + try: + return process_workbook(wb=reader.wb) + finally: + reader.wb.close() + reader.archive.close() + except (OSError, BadZipFile, KeyError) as read_err: + raise PyXFormError("Error reading .xlsx file: %s" % read_err) from read_err def xlsx_value_to_str(value) -> str: @@ -311,7 +344,7 @@ def replace_prefix(d, prefix): elif isinstance(v, dict): d[k] = replace_prefix(v, prefix) elif isinstance(v, list): - d[k] = map(lambda x: replace_prefix(x, prefix), v) + d[k] = (replace_prefix(x, prefix) for x in v) return d return_list.append(replace_prefix(row["lambda"], prefix)) @@ -326,7 +359,7 @@ def replace_prefix(d, prefix): def csv_to_dict(path_or_file): if isinstance(path_or_file, str): - csv_data = open(path_or_file, "r", encoding="utf-8", newline="") + csv_data = open(path_or_file, encoding="utf-8", newline="") else: csv_data = path_or_file @@ -421,7 +454,7 @@ def convert_file_to_csv_string(path): for out_key in out_keys: out_row.append(row.get(out_key, None)) out_rows.append(out_row) - writer.writerow([None] + out_keys) + writer.writerow([None, *out_keys]) for out_row in out_rows: - writer.writerow([None] + out_row) + writer.writerow([None, *out_row]) return foo.getvalue() diff --git a/pyxform/xls2xform.py b/pyxform/xls2xform.py index feb6e75d5..e31d5e1af 100644 --- a/pyxform/xls2xform.py +++ b/pyxform/xls2xform.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ xls2xform converts properly formatted Excel documents into XForms for use with ODK Collect. @@ -174,13 +173,13 @@ def main_cli(): pretty_print=args.pretty_print, enketo=args.enketo_validate, ) - except EnvironmentError as e: + except OSError: # Do not crash if 'java' not installed - logger.error(e) - except ODKValidateError as e: + logger.exception("EnvironmentError during conversion") + except ODKValidateError: # Remove output file if there is an error os.remove(args.output_path) - logger.error(e) + logger.exception("ODKValidateError during conversion.") else: if len(warnings) > 0: logger.warning("Warnings:") diff --git a/pyxform/xlsparseutils.py b/pyxform/xlsparseutils.py index 58e773622..7b1c995de 100644 --- a/pyxform/xlsparseutils.py +++ b/pyxform/xlsparseutils.py @@ -7,7 +7,7 @@ # http://www.w3.org/TR/REC-xml/ TAG_START_CHAR = r"[a-zA-Z:_]" TAG_CHAR = r"[a-zA-Z:_0-9\-.]" -XFORM_TAG_REGEXP = "%(start)s%(char)s*" % {"start": TAG_START_CHAR, "char": TAG_CHAR} +XFORM_TAG_REGEXP = f"{TAG_START_CHAR}{TAG_CHAR}*" def find_sheet_misspellings(key: str, keys: "KeysView") -> "Optional[str]": @@ -32,7 +32,7 @@ def find_sheet_misspellings(key: str, keys: "KeysView") -> "Optional[str]": msg = ( "When looking for a sheet named '{k}', the following sheets with " "similar names were found: {c}." - ).format(k=key, c=str(", ".join(("'{}'".format(c) for c in candidates)))) + ).format(k=key, c=str(", ".join(f"'{c}'" for c in candidates))) return msg else: return None diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d3eef26d6..000000000 --- a/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -# Make sure line length matches pyproject.toml setting for black -max-line-length=90 -ignore=E203,E501,W503 - -[pycodestyle] -# Make sure line length matches pyproject.toml setting for black -max-line-length=90 -ignore=E203,E501,W503 diff --git a/setup.py b/setup.py deleted file mode 100644 index fec5a1fdf..000000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -""" -pyxform - Python library that converts XLSForms to XForms. -""" -from setuptools import find_packages, setup - - -setup( - name="pyxform", - version="2.0.2", - author="github.com/xlsform", - author_email="info@xlsform.org", - packages=find_packages(exclude=["tests", "tests.*"]), - package_data={ - "pyxform.validators.odk_validate": ["bin/*.*"], - "pyxform": ["iana_subtags.txt"], - }, - url="http://pypi.python.org/pypi/pyxform/", - description="A Python package to create XForms for ODK Collect.", - long_description=open("README.rst", "rt").read(), - python_requires=">=3.7", - install_requires=[ - "xlrd==2.0.1", - "openpyxl==3.0.9", - "defusedxml==0.7.1", - ], - entry_points={ - "console_scripts": [ - "xls2xform=pyxform.xls2xform:main_cli", - "pyxform_validator_update=pyxform.validators.updater:main_cli", - ] - }, -) diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 3849ac523..31e433db1 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- """ PyxformTestCase base class using markdown to define the XLSForm. """ -import codecs import logging import os import re @@ -15,12 +13,12 @@ # noinspection PyProtectedMember from lxml.etree import _Element - from pyxform.builder import create_survey_element_from_dict from pyxform.errors import PyXFormError from pyxform.utils import NSMAP, coalesce from pyxform.validators.odk_validate import ODKValidateError, check_xform from pyxform.xls2json import workbook_to_json + from tests.test_utils.md_table import md_table_to_ss_structure logger = logging.getLogger(__name__) @@ -57,7 +55,7 @@ def md_to_pyxform_survey( id_string: Optional[str] = None, debug: bool = False, autoname: bool = True, - warnings: List[str] = None, + warnings: Optional[List[str]] = None, ): if autoname: kwargs = self._autoname_inputs(name=name, title=title, id_string=id_string) @@ -85,7 +83,7 @@ def list_to_dicts(arr): def _row_to_dict(row): out_dict = {} - for i in range(0, len(row)): + for i in range(len(row)): col = row[i] if col not in [None, ""]: out_dict[headers[i]] = col @@ -111,7 +109,7 @@ def _ss_structure_to_pyxform_survey( name: Optional[str] = None, title: Optional[str] = None, id_string: Optional[str] = None, - warnings: List[str] = None, + warnings: Optional[List[str]] = None, ): # using existing methods from the builder imported_survey_json = workbook_to_json( @@ -132,14 +130,15 @@ def _run_odk_validate(xml): tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) tmp.close() try: - with codecs.open(tmp.name, mode="w", encoding="utf-8") as fp: + with open(tmp.name, mode="w", encoding="utf-8") as fp: fp.write(xml) fp.close() check_xform(tmp.name) finally: # Clean up the temporary file os.remove(tmp.name) - assert not os.path.isfile(tmp.name) + if os.path.isfile(tmp.name): + raise PyXFormError(f"Temporary file still exists: {tmp.name}") @staticmethod def _autoname_inputs( @@ -163,34 +162,34 @@ class PyxformTestCase(PyxformMarkdown, TestCase): def assertPyxformXform( self, # Survey input - md: str = None, - ss_structure: Dict = None, - survey: "Survey" = None, + md: Optional[str] = None, + ss_structure: Optional[Dict] = None, + survey: Optional["Survey"] = None, # XForm assertions - xml__xpath_match: Iterable[str] = None, - xml__xpath_exact: Iterable[Tuple[str, Set[str]]] = None, - xml__xpath_count: Iterable[Tuple[str, int]] = None, + xml__xpath_match: Optional[Iterable[str]] = None, + xml__xpath_exact: Optional[Iterable[Tuple[str, Set[str]]]] = None, + xml__xpath_count: Optional[Iterable[Tuple[str, int]]] = None, # XForm assertions - deprecated - xml__contains: Iterable[str] = None, - xml__excludes: Iterable[str] = None, - model__contains: Iterable[str] = None, - model__excludes: Iterable[str] = None, - itext__contains: Iterable[str] = None, - itext__excludes: Iterable[str] = None, - instance__contains: Iterable[str] = None, + xml__contains: Optional[Iterable[str]] = None, + xml__excludes: Optional[Iterable[str]] = None, + model__contains: Optional[Iterable[str]] = None, + model__excludes: Optional[Iterable[str]] = None, + itext__contains: Optional[Iterable[str]] = None, + itext__excludes: Optional[Iterable[str]] = None, + instance__contains: Optional[Iterable[str]] = None, # Errors assertions - error__contains: Iterable[str] = None, - error__not_contains: Iterable[str] = None, - odk_validate_error__contains: Iterable[str] = None, - warnings__contains: Iterable[str] = None, - warnings__not_contains: Iterable[str] = None, + error__contains: Optional[Iterable[str]] = None, + error__not_contains: Optional[Iterable[str]] = None, + odk_validate_error__contains: Optional[Iterable[str]] = None, + warnings__contains: Optional[Iterable[str]] = None, + warnings__not_contains: Optional[Iterable[str]] = None, warnings_count: Optional[int] = None, errored: bool = False, # Optional extras - name: str = None, - id_string: str = None, - title: str = None, - warnings: List[str] = None, + name: Optional[str] = None, + id_string: Optional[str] = None, + title: Optional[str] = None, + warnings: Optional[List[str]] = None, run_odk_validate: bool = False, debug: bool = False, ): @@ -319,13 +318,13 @@ def _pull_xml_node_from_root(element_selector): "Expected valid survey but compilation failed. Try correcting the " "error with 'debug=True', setting 'errored=True', and or optionally " "'error__contains=[...]'\nError(s): " + "\n".join(errors) - ) + ) from e except ODKValidateError as e: if not odk_validate_error__contains: raise PyxformTestError( "ODK Validate error was thrown but 'odk_validate_error__contains' " "was empty: " + str(e) - ) + ) from e for v_err in odk_validate_error__contains: self.assertContains( e.args[0], v_err, msg_prefix="odk_validate_error__contains" @@ -491,7 +490,7 @@ def assert_xpath_exact( """ if not (isinstance(xpath, str) and isinstance(expected, set)): msg = "Each xpath_exact requires: tuple(xpath: str, expected: Set[str])." - raise SyntaxError(msg) + raise TypeError(msg) observed = xpath_evaluate( matcher_context=matcher_context, content=content, @@ -523,7 +522,7 @@ def assert_xpath_count( """ if not (isinstance(xpath, str) and isinstance(expected, int)): msg = "Each xpath_count requires: tuple(xpath: str, count: int)" - raise SyntaxError(msg) + raise TypeError(msg) observed = xpath_evaluate( matcher_context=matcher_context, content=content, diff --git a/tests/test_allow_mock_accuracy.py b/tests/test_allow_mock_accuracy.py index e5952040c..bb58c821b 100644 --- a/tests/test_allow_mock_accuracy.py +++ b/tests/test_allow_mock_accuracy.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_area.py b/tests/test_area.py index d6483c74a..10cf6360f 100644 --- a/tests/test_area.py +++ b/tests/test_area.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ AreaTest - test enclosed-area(geo_shape) calculation. """ diff --git a/tests/test_audio_quality.py b/tests/test_audio_quality.py index 36f28f338..02ef21b3a 100644 --- a/tests/test_audio_quality.py +++ b/tests/test_audio_quality.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_audit.py b/tests/test_audit.py index 36b5f19f6..6ef65a292 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ AuditTest - test audit question type. """ diff --git a/tests/test_background_audio.py b/tests/test_background_audio.py index 9795657b3..ca4a6e574 100644 --- a/tests/test_background_audio.py +++ b/tests/test_background_audio.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_bind_conversions.py b/tests/test_bind_conversions.py index 70581be81..d6d58016c 100644 --- a/tests/test_bind_conversions.py +++ b/tests/test_bind_conversions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ BindConversionsTest - test bind conversions. """ diff --git a/tests/test_bug_missing_headers.py b/tests/test_bug_missing_headers.py index 8fdaf58f5..7013d8719 100644 --- a/tests/test_bug_missing_headers.py +++ b/tests/test_bug_missing_headers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test missing headers in XLSForm. """ diff --git a/tests/test_bug_round_calculation.py b/tests/test_bug_round_calculation.py index 8627ce333..64f52e352 100644 --- a/tests/test_bug_round_calculation.py +++ b/tests/test_bug_round_calculation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test round(number, precision) calculation. """ @@ -14,6 +13,6 @@ def test_non_existent_itext_reference(self): | | type | name | label | calculation | | | decimal | amount | Counter | | | | calculate | rounded | Rounded | round(${amount}, 0) | - """, # noqa + """, xml__contains=[""""""], ) diff --git a/tests/builder_tests.py b/tests/test_builder.py similarity index 98% rename from tests/builder_tests.py rename to tests/test_builder.py index 19cfaed3f..a230d46ea 100644 --- a/tests/builder_tests.py +++ b/tests/test_builder.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- """ Test builder module functionality. """ import os import re -import xml.etree.ElementTree as ETree from unittest import TestCase +import defusedxml.ElementTree as ETree from pyxform import InputQuestion, Survey from pyxform.builder import SurveyElementBuilder, create_survey_from_xls from pyxform.errors import PyXFormError from pyxform.xls2json import print_pyobj_to_json + from tests import utils FIXTURE_FILETYPE = "xls" @@ -549,12 +549,9 @@ def test_style_not_added_to_body_if_not_present(self): xml = survey.to_xml() # find the body tag root_elm = ETree.fromstring(xml.encode("utf-8")) - body_elms = list( - filter( - lambda e: self.STRIP_NS_FROM_TAG_RE.sub("", e.tag) == "body", - [c for c in root_elm], - ) - ) + body_elms = [ + c for c in root_elm if self.STRIP_NS_FROM_TAG_RE.sub("", c.tag) == "body" + ] self.assertEqual(len(body_elms), 1) self.assertIsNone(body_elms[0].get("class")) @@ -565,11 +562,8 @@ def test_style_added_to_body_if_present(self): xml = survey.to_xml() # find the body tag root_elm = ETree.fromstring(xml.encode("utf-8")) - body_elms = list( - filter( - lambda e: self.STRIP_NS_FROM_TAG_RE.sub("", e.tag) == "body", - [c for c in root_elm], - ) - ) + body_elms = [ + c for c in root_elm if self.STRIP_NS_FROM_TAG_RE.sub("", c.tag) == "body" + ] self.assertEqual(len(body_elms), 1) self.assertEqual(body_elms[0].get("class"), "ltr") diff --git a/tests/test_choices_sheet.py b/tests/test_choices_sheet.py index e230ca1c5..114ccfc35 100644 --- a/tests/test_choices_sheet.py +++ b/tests/test_choices_sheet.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase from tests.xpath_helpers.choices import xpc from tests.xpath_helpers.questions import xpq diff --git a/tests/test_custom_xml_namespaces.py b/tests/test_custom_xml_namespaces.py index 09bf9d4f0..57933eb5f 100644 --- a/tests/test_custom_xml_namespaces.py +++ b/tests/test_custom_xml_namespaces.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test custom namespaces. """ diff --git a/tests/dump_and_load_tests.py b/tests/test_dump_and_load.py similarity index 90% rename from tests/dump_and_load_tests.py rename to tests/test_dump_and_load.py index 8bdd2089b..5ddb2d035 100644 --- a/tests/dump_and_load_tests.py +++ b/tests/test_dump_and_load.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test multiple XLSForm can be generated successfully. """ @@ -6,6 +5,7 @@ from unittest import TestCase from pyxform.builder import create_survey_from_path + from tests import utils @@ -32,13 +32,13 @@ def setUp(self): self.surveys[filename] = create_survey_from_path(path) def test_load_from_dump(self): - for filename, survey in self.surveys.items(): + for survey in self.surveys.values(): survey.json_dump() path = survey.name + ".json" survey_from_dump = create_survey_from_path(path) self.assertEqual(survey.to_json_dict(), survey_from_dump.to_json_dict()) def tearDown(self): - for filename, survey in self.surveys.items(): + for survey in self.surveys.values(): path = survey.name + ".json" os.remove(path) diff --git a/tests/test_dynamic_default.py b/tests/test_dynamic_default.py index cd6642291..79644f5df 100644 --- a/tests/test_dynamic_default.py +++ b/tests/test_dynamic_default.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test handling dynamic default in forms """ @@ -10,8 +9,8 @@ from unittest.mock import patch import psutil - from pyxform import utils + from tests.pyxform_test_case import PyxformTestCase from tests.xpath_helpers.choices import xpc from tests.xpath_helpers.questions import xpq @@ -45,7 +44,7 @@ class XPathHelper: @staticmethod def model_setvalue(q_num: int): """Get the setvalue element's value attribute.""" - return fr""" + return rf""" /h:html/h:head/x:model/x:setvalue[ @ref="/test_name/q{q_num}" and @event='odk-instance-first-load' @@ -79,7 +78,7 @@ def model(q_num: int, case: Case): value_cmp = "" else: value_cmp = f"""and @value="{q_default_final}" """ - return fr""" + return rf""" /h:html/h:head/x:model /x:instance/x:test_name[@id="test_id"]/x:q{q_num}[ not(text()) @@ -97,12 +96,11 @@ def model(q_num: int, case: Case): else: if 0 == len(q_default_final): q_default_cmp = """and not(text()) """ + elif "'" in q_default_final: + q_default_cmp = "" else: - if "'" in q_default_final: - q_default_cmp = "" - else: - q_default_cmp = f"""and text()='{q_default_final}' """ - return fr""" + q_default_cmp = f"""and text()='{q_default_final}' """ + return rf""" /h:html/h:head/x:model /x:instance/x:test_name[@id="test_id"]/x:q{q_num}[ ancestor::x:model/x:bind[ @@ -137,7 +135,7 @@ def body_select1(q_num: int, choices: Tuple[Tuple[str, str], ...]): for cv, cl in choices ) ) - return fr""" + return rf""" /h:html/h:body/x:select1[ @ref = '/test/q{q_num}' and ./x:label/text() = 'Select{q_num}' @@ -804,23 +802,23 @@ def test_dynamic_default_performance__time(self): | | text | q{i} | Q{i} | if(../t2 = 'test', 1, 2) + 15 - int(1.2) | """ for count in (500, 1000, 2000): - questions = "\n".join((question.format(i=i) for i in range(1, count))) + questions = "\n".join(question.format(i=i) for i in range(1, count)) md = "".join((survey_header, questions)) - def run(name): + def run(name, case): runs = 0 results = [] while runs < 10: start = perf_counter() - self.assertPyxformXform(md=md) + self.assertPyxformXform(md=case) results.append(perf_counter() - start) runs += 1 print(name, round(sum(results) / len(results), 4)) - run(name=f"questions={count}, with check (seconds):") + run(name=f"questions={count}, with check (seconds):", case=md) with patch("pyxform.utils.default_is_dynamic", return_value=True): - run(name=f"questions={count}, without check (seconds):") + run(name=f"questions={count}, without check (seconds):", case=md) def test_dynamic_default_performance__memory(self): """ @@ -838,7 +836,7 @@ def test_dynamic_default_performance__memory(self): question = """ | | text | q{i} | Q{i} | if(../t2 = 'test', 1, 2) + 15 - int(1.2) | """ - questions = "\n".join((question.format(i=i) for i in range(1, 2000))) + questions = "\n".join(question.format(i=i) for i in range(1, 2000)) md = "".join((survey_header, questions)) process = psutil.Process(os.getpid()) pre_mem = process.memory_info().rss diff --git a/tests/test_entities_create.py b/tests/test_entities_create.py index bd3d9055e..2a39b656e 100644 --- a/tests/test_entities_create.py +++ b/tests/test_entities_create.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_entities_update.py b/tests/test_entities_update.py index 5e326607b..9ec7616e9 100644 --- a/tests/test_entities_update.py +++ b/tests/test_entities_update.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from tests.pyxform_test_case import PyxformTestCase diff --git a/tests/test_expected_output/attribute_columns_test.xml b/tests/test_expected_output/attribute_columns_test.xml index 89071bd05..61c895e3d 100644 --- a/tests/test_expected_output/attribute_columns_test.xml +++ b/tests/test_expected_output/attribute_columns_test.xml @@ -3,15 +3,24 @@ Spec test - + - - - + + Yes + + + No - + jr://images/a.jpg + + jr://images/b.jpg + + + - + - @@ -36,9 +45,6 @@ - - - Yes - - @@ -48,9 +54,6 @@ - - - Yes - - @@ -66,12 +69,6 @@ - - - No - - - jr://images/a.jpg - - @@ -93,21 +90,9 @@ - - - jr://images/b.jpg - - - Yes - - - No - - - - Yes - - @@ -132,9 +117,6 @@ - - - No - - @@ -144,9 +126,6 @@ - - - No - - @@ -156,45 +135,18 @@ - - - No - - - - Yes - - - Yes - - - jr://images/b.jpg - - - Yes - - - No - - a note - - Yes - - - No - jr://images/img_test.jpg - - No - - @@ -209,6 +161,14 @@ + + 是 + + + 没有 + + + - @@ -233,15 +193,9 @@ - - - jr://images/- - ni hao - - 是 - - @@ -251,16 +205,11 @@ - - - 是 - - - jr://images/- jr://audio/chinese_audio.wav - jr://video/- 您好 @@ -269,12 +218,6 @@ - - - 没有 - - - jr://images/- - - @@ -296,30 +239,13 @@ - - - jr://images/- - - - 是 - - - 没有 - - - 没有 - - - 是 - - - - - jr://images/- - + - @@ -335,9 +261,6 @@ - - - 没有 - - @@ -347,9 +270,6 @@ - - - 没有 - - @@ -365,39 +285,15 @@ - - - 是 - - - 是 - - - jr://images/- - - - 是 - - - 没有 - - - - - 是 - - - 没有 - - - - 没有 - - @@ -412,6 +308,14 @@ + + - + + + - + + + video_test @@ -427,9 +331,6 @@ constrained decimal - - - - This launches a fictional application to get an integer result. @@ -439,9 +340,6 @@ select multiple test - - - - deviceid_test_output: @@ -452,16 +350,11 @@ start test output: - - - - label-test jr://images/img_test_2.jpg - jr://audio/- - jr://video/- text_image_audio_video_test @@ -469,12 +362,6 @@ date_test - - - - - - jr://images/- - autocomplete_test @@ -483,44 +370,24 @@ - - - - - image_test datetime_test - - jr://images/- - today_test_output: - - - - audio_test - - jr://images/- - - - - - - - - - a integer - - jr://images/- - + labeled select group test @@ -533,17 +400,11 @@ Skip to end - - - - Your name is time_test - - - - simserial_test_output: @@ -563,33 +424,12 @@ - - - - - - - - - - - jr://images/- - - - - - - - - - You entered an email address - - - - - - - - - required_text @@ -623,7 +463,23 @@ - + + + + + + + + + + + + + + + + + @@ -675,6 +531,30 @@ + + + + yes_no-0 + yes + + + yes_no-1 + no + + + + + + + a_b-0 + a + + + a_b-1 + b + + + @@ -734,14 +614,10 @@ @@ -771,25 +647,17 @@ - - - - + + + diff --git a/tests/test_external_instances.py b/tests/test_external_instances.py index 385cb8323..1f072d0c9 100644 --- a/tests/test_external_instances.py +++ b/tests/test_external_instances.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xml-external syntax and instances generated from pulldata calls. @@ -7,6 +6,7 @@ from textwrap import dedent from pyxform.errors import PyXFormError + from tests.pyxform_test_case import PyxformTestCase, PyxformTestError from tests.xpath_helpers.choices import xpc @@ -171,7 +171,7 @@ def test_cannot__use_same_external_xml_id_with_mixed_types(self): | | begin group | g4 | | | | | calculate | city | City | pulldata('fruits', 'name', 'name', 'mango') | | | end group | g4 | | | - """, # noqa + """, model__contains=[], ) self.assertIn("The name 'city' was found 2 time(s)", repr(ctx.exception)) @@ -196,7 +196,7 @@ def test_can__use_same_external_csv_id_with_mixed_types(self): | | text | foo | Foo | | | | calculate | city | City | pulldata('fruits', 'name', 'name', 'mango') | | | end group | g4 | | | - """, # noqa + """, model__contains=[''], ) @@ -225,12 +225,12 @@ def test_can__use_all_types_together_with_unique_ids(self): | | list_name | name | label | | | | | states | 1 | Pass | | | | | states | 2 | Fail | | | - """, # noqa + """, model__contains=[ '', '', '', - ], # noqa + ], xml__xpath_match=[ xpc.model_instance_choices_label("states", (("1", "Pass"), ("2", "Fail"))) ], @@ -247,7 +247,7 @@ def test_cannot__use_different_src_same_id__select_then_internal(self): | | list_name | name | label | | | | states | 1 | Pass | | | | states | 2 | Fail | | - """ # noqa + """ with self.assertRaises(PyXFormError) as ctx: survey = self.md_to_pyxform_survey(md_raw=md) survey._to_pretty_xml() @@ -271,14 +271,14 @@ def test_cannot__use_different_src_same_id__external_then_pulldata(self): | | calculate | f_csv | City | pulldata('fruits', 'name', 'name', 'mango') | | | note | note | Fruity! ${f_csv} | | | | end group | g1 | | | - """ # noqa + """ with self.assertRaises(PyXFormError) as ctx: survey = self.md_to_pyxform_survey(md_raw=md) survey._to_pretty_xml() self.assertIn( "Instance name: 'fruits', " "Existing type: 'external', Existing URI: 'jr://file/fruits.xml', " - "Duplicate type: 'pulldata', Duplicate URI: 'jr://file-csv/fruits.csv', " # noqa + "Duplicate type: 'pulldata', Duplicate URI: 'jr://file-csv/fruits.csv', " "Duplicate context: '[type: group, name: g1]'.", repr(ctx.exception), ) @@ -295,14 +295,14 @@ def test_cannot__use_different_src_same_id__pulldata_then_external(self): | | xml-external | fruits | | | | | note | note | Fruity! ${f_csv} | | | | end group | g1 | | | - """ # noqa + """ with self.assertRaises(PyXFormError) as ctx: survey = self.md_to_pyxform_survey(md_raw=md) survey._to_pretty_xml() self.assertIn( "Instance name: 'fruits', " - "Existing type: 'pulldata', Existing URI: 'jr://file-csv/fruits.csv', " # noqa - "Duplicate type: 'external', Duplicate URI: 'jr://file/fruits.xml', " # noqa + "Existing type: 'pulldata', Existing URI: 'jr://file-csv/fruits.csv', " + "Duplicate type: 'external', Duplicate URI: 'jr://file/fruits.xml', " "Duplicate context: '[type: group, name: g1]'.", repr(ctx.exception), ) @@ -318,10 +318,10 @@ def test_can__reuse_csv__selects_then_pulldata(self): | | select_one_from_file pain_locations.csv | pyear | Location of worst pain this year. | | | | calculate | f_csv | pd | pulldata('pain_locations', 'name', 'name', 'arm') | | | note | note | Arm ${f_csv} | | - """ # noqa + """ expected = """ -""" # noqa +""" self.assertPyxformXform(md=md, model__contains=[expected]) survey = self.md_to_pyxform_survey(md_raw=md) xml = survey._to_pretty_xml() @@ -338,8 +338,10 @@ def test_can__reuse_csv__pulldata_then_selects(self): | | select_one_from_file pain_locations.csv | pweek | Location of worst pain this week. | | | | select_one_from_file pain_locations.csv | pmonth | Location of worst pain this month. | | | | select_one_from_file pain_locations.csv | pyear | Location of worst pain this year. | | - """ # noqa - expected = """""" # noqa + """ + expected = ( + """""" + ) self.assertPyxformXform(md=md, model__contains=[expected]) def test_can__reuse_xml__selects_then_external(self): @@ -352,10 +354,10 @@ def test_can__reuse_xml__selects_then_external(self): | | select_one_from_file pain_locations.xml | pmonth | Location of worst pain this month. | | | select_one_from_file pain_locations.xml | pyear | Location of worst pain this year. | | | xml-external | pain_locations | | - """ # noqa + """ expected = """ -""" # noqa +""" survey = self.md_to_pyxform_survey(md_raw=md) xml = survey._to_pretty_xml() self.assertEqual(1, xml.count(expected)) @@ -370,8 +372,10 @@ def test_can__reuse_xml__external_then_selects(self): | | select_one_from_file pain_locations.xml | pweek | Location of worst pain this week. | | | select_one_from_file pain_locations.xml | pmonth | Location of worst pain this month. | | | select_one_from_file pain_locations.xml | pyear | Location of worst pain this year. | - """ # noqa - expected = """""" # noqa + """ + expected = ( + """""" + ) self.assertPyxformXform(md=md, model__contains=[expected]) survey = self.md_to_pyxform_survey(md_raw=md) xml = survey._to_pretty_xml() @@ -602,7 +606,7 @@ def test_mixed_quotes_and_functions_in_pulldata(self): | | calculate | calculate2 | | pulldata('instance2',"last",'rcid',concat('RC',${rcid})) | | | calculate | calculate3 | | pulldata('instance3','envelope','rcid',"Bar") | | | calculate | calculate4 | | pulldata('instance4' ,'envelope','rcid',"Bar") | - """, # noqa + """, xml__contains=[ '', '', diff --git a/tests/test_external_instances_for_selects.py b/tests/test_external_instances_for_selects.py index 028ada445..32b78b280 100644 --- a/tests/test_external_instances_for_selects.py +++ b/tests/test_external_instances_for_selects.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test external instance syntax @@ -11,6 +10,7 @@ from pyxform.constants import EXTERNAL_INSTANCE_EXTENSIONS from pyxform.errors import PyXFormError from pyxform.xls2xform import get_xml_path, xls2xform_convert + from tests.pyxform_test_case import PyxformTestCase from tests.test_utils.md_table import md_table_to_workbook from tests.utils import get_temp_dir @@ -252,7 +252,7 @@ def test_param_value_case_preserved(self): | | type | name | label | parameters | | | select_one_from_file cities{ext} | city | City | value=VAL, label=lBl | """ - for ext, xp_city, xp_subs in self.xp_test_args: + for ext, xp_city, _ in self.xp_test_args: with self.subTest(msg=ext): self.assertPyxformXform( name="test", @@ -441,7 +441,7 @@ def test_itemset_csv_generated_from_external_choices(self): self.assertIn(log_msg, [r.message for r in log.records]) self.assertTrue(os.path.exists(itemsets_path)) - with open(itemsets_path, "r") as csv: + with open(itemsets_path, encoding="utf-8") as csv: rows = csv.readlines() # Should have the non-empty headers in the first row. self.assertEqual('"list_name","name","state","city"\n', rows[0]) diff --git a/tests/test_fieldlist_labels.py b/tests/test_fieldlist_labels.py index 448e05a90..bf3a4badc 100644 --- a/tests/test_fieldlist_labels.py +++ b/tests/test_fieldlist_labels.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test field-list labels """ diff --git a/tests/test_fields.py b/tests/test_fields.py index a1df844c1..86afb9c7c 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test duplicate survey question field name. """ @@ -59,7 +58,7 @@ def test_duplicate_choices_without_setting(self): errored=True, error__contains=[ "The name column for the 'list' choice list contains these duplicates: 'b'" - ], # noqa + ], ) def test_multiple_duplicate_choices_without_setting(self): @@ -78,7 +77,7 @@ def test_multiple_duplicate_choices_without_setting(self): errored=True, error__contains=[ "The name column for the 'list' choice list contains these duplicates: 'a', 'b'" - ], # noqa + ], ) def test_duplicate_choices_with_setting_not_set_to_yes(self): @@ -99,7 +98,7 @@ def test_duplicate_choices_with_setting_not_set_to_yes(self): errored=True, error__contains=[ "The name column for the 'list' choice list contains these duplicates: 'b'" - ], # noqa + ], ) def test_duplicate_choices_with_allow_choice_duplicates_setting(self): diff --git a/tests/test_file.py b/tests/test_file.py index 67c7f3678..21970c010 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test file question type. """ diff --git a/tests/file_utils_test.py b/tests/test_file_utils.py similarity index 96% rename from tests/file_utils_test.py rename to tests/test_file_utils.py index a697ed4c7..df50d123e 100644 --- a/tests/file_utils_test.py +++ b/tests/test_file_utils.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- """ Test xls2json_backends util functions. """ from unittest import TestCase from pyxform.xls2json_backends import convert_file_to_csv_string + from tests import utils diff --git a/tests/test_for_loop.py b/tests/test_for_loop.py index a068f12d0..03f7d2585 100644 --- a/tests/test_for_loop.py +++ b/tests/test_for_loop.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test loop question type. """ diff --git a/tests/test_form_name.py b/tests/test_form_name.py index 79dd78f0f..36df4edb0 100644 --- a/tests/test_form_name.py +++ b/tests/test_form_name.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test setting form name to data. """ @@ -24,7 +23,7 @@ def test_default_to_data_when_no_name(self): # We're passing autoname false when creating the survey object. self.assertEqual(survey.id_string, None) - self.assertEqual(survey.name, str("data")) + self.assertEqual(survey.name, "data") self.assertEqual(survey.title, None) # Set required fields because we need them if we want to do xml comparison. diff --git a/tests/test_geo.py b/tests/test_geo.py index 00f412b9f..548295e54 100644 --- a/tests/test_geo.py +++ b/tests/test_geo.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test geo widgets. """ diff --git a/tests/group_test.py b/tests/test_group.py similarity index 99% rename from tests/group_test.py rename to tests/test_group.py index 37fb69527..68597cc76 100644 --- a/tests/group_test.py +++ b/tests/test_group.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing simple cases for Xls2Json """ @@ -6,6 +5,7 @@ from pyxform.builder import create_survey_element_from_dict from pyxform.xls2json import SurveyReader + from tests import utils diff --git a/tests/test_groups.py b/tests/test_groups.py index 21b223aea..5853fcfee 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XForm groups. """ diff --git a/tests/test_guidance_hint.py b/tests/test_guidance_hint.py index d292907f9..8566a9c48 100644 --- a/tests/test_guidance_hint.py +++ b/tests/test_guidance_hint.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Guidance hint test module. """ @@ -28,7 +27,7 @@ def test_guidance_hint_and_label(self): | survey | | | | | | | type | name | label | guidance_hint | | | string | name | Name | as shown on birth certificate| - """, # noqa + """, xml__contains=[ "", 'as shown on birth certificate', @@ -44,7 +43,7 @@ def test_hint_and_guidance_one_language(self): # pylint: disable=C0103 | survey | | | | | | | | type | name | label | hint | guidance_hint | | | string | name | Name | your name | as shown on birth certificate| - """, # noqa + """, xml__contains=[ "", "your name", @@ -60,10 +59,10 @@ def test_multi_language_guidance(self): | survey | | | | | | | | | type | name | label | hint | guidance_hint | guidance_hint::French (fr) | | | string | name | Name | your name | as shown on birth certificate| comme sur le certificat de naissance| - """, # noqa + """, xml__contains=[ '', - 'comme sur le certificat de naissance', # noqa + 'comme sur le certificat de naissance', '', 'as shown on birth certificate', "", @@ -91,7 +90,7 @@ def test_multi_language_guidance_only(self): # pylint:disable=C0103 | survey | | | | | | | type | name | guidance_hint | guidance_hint::French (fr) | | | string | name | as shown on birth certificate| comme sur le certificat de naissance | - """, # noqa + """, errored=True, error__contains=["The survey element named 'name' has no label or hint."], ) @@ -104,7 +103,7 @@ def test_multi_language_hint(self): | survey | | | | | | | type | name | hint | hint::French (fr) | | | string | name | default language hint| French hint | - """, # noqa + """, xml__contains=[ "", "French hint", diff --git a/tests/test_image_app_parameter.py b/tests/test_image_app_parameter.py index 6f6324f5d..36f55bab5 100644 --- a/tests/test_image_app_parameter.py +++ b/tests/test_image_app_parameter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test image max-pixels and app parameters. """ diff --git a/tests/j2x_test_creation.py b/tests/test_j2x_creation.py similarity index 97% rename from tests/j2x_test_creation.py rename to tests/test_j2x_creation.py index 0843ee009..e8f79ad3c 100644 --- a/tests/j2x_test_creation.py +++ b/tests/test_j2x_creation.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- """ Testing creation of Surveys using verbose methods """ from unittest import TestCase from pyxform import MultipleChoiceQuestion, Survey, create_survey_from_xls + from tests import utils @@ -89,4 +89,4 @@ def allow_surveys_with_comment_rows(self): }, "type": "survey", } - self.assertEquals(survey.to_json_dict(), expected_dict) + self.assertEqual(survey.to_json_dict(), expected_dict) diff --git a/tests/j2x_test_instantiation.py b/tests/test_j2x_instantiation.py similarity index 90% rename from tests/j2x_test_instantiation.py rename to tests/test_j2x_instantiation.py index 5749badc8..0b419be14 100644 --- a/tests/j2x_test_instantiation.py +++ b/tests/test_j2x_instantiation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing the instance object for pyxform. """ @@ -6,6 +5,7 @@ from pyxform import Survey, SurveyInstance from pyxform.builder import create_survey_element_from_dict + from tests.utils import prep_class_config @@ -26,8 +26,8 @@ def test_simple_survey_instantiation(self): i = surv.instantiate() - self.assertEquals(i.keys(), ["survey_question"]) - self.assertEquals(set(i.xpaths()), {"/Simple", "/Simple/survey_question"}) + self.assertEqual(i.keys(), ["survey_question"]) + self.assertEqual(set(i.xpaths()), {"/Simple", "/Simple/survey_question"}) def test_simple_survey_answering(self): surv = Survey(name="Water") @@ -43,10 +43,10 @@ def test_simple_survey_answering(self): i = SurveyInstance(surv) i.answer(name="color", value="blue") - self.assertEquals(i.answers()["color"], "blue") + self.assertEqual(i.answers()["color"], "blue") i.answer(name="feeling", value="liquidy") - self.assertEquals(i.answers()["feeling"], "liquidy") + self.assertEqual(i.answers()["feeling"], "liquidy") def test_answers_can_be_imported_from_xml(self): surv = Survey(name="data") diff --git a/tests/j2x_question_tests.py b/tests/test_j2x_question.py similarity index 99% rename from tests/j2x_question_tests.py rename to tests/test_j2x_question.py index f04657ba2..7ba69f9ec 100644 --- a/tests/j2x_question_tests.py +++ b/tests/test_j2x_question.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing creation of Surveys using verbose methods """ @@ -6,6 +5,7 @@ from pyxform import Survey from pyxform.builder import create_survey_element_from_dict + from tests.utils import prep_class_config TESTING_BINDINGS = True @@ -177,7 +177,7 @@ def test_simple_phone_number_question_type_multilingual(self): "type": "string", "constraint": r"regex(., '^\d*$')", } - observed = {k: v for k, v in q.xml_bindings()[0].attributes.items()} + observed = {k: v for k, v in q.xml_bindings()[0].attributes.items()} # noqa: C416 self.assertDictEqual(expected, observed) def test_simple_select_all_question_multilingual(self): diff --git a/tests/j2x_test_xform_build_preparation.py b/tests/test_j2x_xform_build_preparation.py similarity index 98% rename from tests/j2x_test_xform_build_preparation.py rename to tests/test_j2x_xform_build_preparation.py index cec5f4f79..3740c908b 100644 --- a/tests/j2x_test_xform_build_preparation.py +++ b/tests/test_j2x_xform_build_preparation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing preparation of values for XForm exporting """ @@ -9,7 +8,6 @@ class Json2XformExportingPrepTests(TestCase): def test_dictionary_consolidates_duplicate_entries(self): - yes_or_no_dict_array = [ {"label": {"French": "Oui", "English": "Yes"}, "name": "yes"}, {"label": {"French": "Non", "English": "No"}, "name": "no"}, diff --git a/tests/js2x_test_import_from_json.py b/tests/test_js2x_import_from_json.py similarity index 98% rename from tests/js2x_test_import_from_json.py rename to tests/test_js2x_import_from_json.py index d3e0ba631..7a698c8dd 100644 --- a/tests/js2x_test_import_from_json.py +++ b/tests/test_js2x_import_from_json.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing our ability to import from a JSON text file. """ diff --git a/tests/json2xform_test.py b/tests/test_json2xform.py similarity index 97% rename from tests/json2xform_test.py rename to tests/test_json2xform.py index bab2d9040..c169debd3 100644 --- a/tests/json2xform_test.py +++ b/tests/test_json2xform.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing simple cases for pyxform """ diff --git a/tests/test_language_warnings.py b/tests/test_language_warnings.py index 5171d9e51..2d17a9a75 100644 --- a/tests/test_language_warnings.py +++ b/tests/test_language_warnings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test language warnings. """ diff --git a/tests/test_last_saved.py b/tests/test_last_saved.py index 5c717c9ef..05f771a02 100644 --- a/tests/test_last_saved.py +++ b/tests/test_last_saved.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ The last-saved virtual instance can be queried to get values from the last saved instance of the form being authored. """ diff --git a/tests/loop_tests.py b/tests/test_loop.py similarity index 97% rename from tests/loop_tests.py rename to tests/test_loop.py index 55b1dfbf3..5a0799500 100644 --- a/tests/loop_tests.py +++ b/tests/test_loop.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- """ Test loop syntax. """ from unittest import TestCase from pyxform.builder import create_survey_from_xls + from tests import utils @@ -89,4 +89,4 @@ def test_loop(self): }, ], } - self.assertEquals(survey.to_json_dict(), expected_dict) + self.assertEqual(survey.to_json_dict(), expected_dict) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index cc9041f5e..0fcd1a8e3 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test language warnings. """ diff --git a/tests/test_osm.py b/tests/test_osm.py index 5f9d3c390..1905a4df2 100644 --- a/tests/test_osm.py +++ b/tests/test_osm.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test OSM widgets. """ diff --git a/tests/test_parameters_rows.py b/tests/test_parameters_rows.py index d857d83ed..035556083 100644 --- a/tests/test_parameters_rows.py +++ b/tests/test_parameters_rows.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test text rows parameter. """ diff --git a/tests/test_pyxform_test_case.py b/tests/test_pyxform_test_case.py index 446e56dc6..8de8fa1fe 100644 --- a/tests/test_pyxform_test_case.py +++ b/tests/test_pyxform_test_case.py @@ -79,10 +79,10 @@ class TestPyxformTestCaseXmlXpath(PyxformTestCase): count=1, ) # Convenience combinations of the above data for Suite 1 tests. - suite1 = [s1c1, s1c2, s1c3, s1c4, s1c5] - suite1_counts = [c.ctuple for c in suite1] - suite1_exacts = [c.etuple for c in suite1] - suite1_xpaths = [c.xpath for c in suite1] + suite1 = (s1c1, s1c2, s1c3, s1c4, s1c5) + suite1_counts = tuple(c.ctuple for c in suite1) + suite1_exacts = tuple(c.etuple for c in suite1) + suite1_xpaths = tuple(c.xpath for c in suite1) # Suite 2: multiple expected match results. # s2c1: element in default namespace. @@ -131,10 +131,10 @@ class TestPyxformTestCaseXmlXpath(PyxformTestCase): count=2, ) # Convenience combinations of the above data for Suite 2 tests. - suite2 = [s2c1, s2c2, s2c3] - suite2_counts = [c.ctuple for c in suite2] - suite2_exacts = [c.etuple for c in suite2] - suite2_xpaths = [c.xpath for c in suite2] + suite2 = (s2c1, s2c2, s2c3) + suite2_counts = tuple(c.ctuple for c in suite2) + suite2_exacts = tuple(c.etuple for c in suite2) + suite2_xpaths = tuple(c.xpath for c in suite2) # Suite 3: other misc cases. # s3c1: XPath with no expected matches. @@ -220,7 +220,7 @@ def test_xml__n_xpath_1_match_fail__xpath_exact(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_exact=self.suite1_exacts + [(self.s1c1.xpath, {"bananas"})], + xml__xpath_exact=[*self.suite1_exacts, (self.s1c1.xpath, {"bananas"})], run_odk_validate=False, ) @@ -229,7 +229,7 @@ def test_xml__n_xpath_1_match_fail__xpath_count(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_count=self.suite1_counts + [(self.s1c1.xpath, 5)], + xml__xpath_count=[*self.suite1_counts, (self.s1c1.xpath, 5)], run_odk_validate=False, ) @@ -238,7 +238,7 @@ def test_xml__n_xpath_1_match_fail__xpath_match(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_match=self.suite1_xpaths + [self.s3c1.xpath], + xml__xpath_match=[*self.suite1_xpaths, self.s3c1.xpath], run_odk_validate=False, ) @@ -314,8 +314,10 @@ def test_xml__n_xpath_n_match_fail__xpath_exact(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_exact=self.suite2_exacts - + [(self.s2c3.xpath, {"bananas", "eggs"})], + xml__xpath_exact=[ + *self.suite2_exacts, + (self.s2c3.xpath, {"bananas", "eggs"}), + ], run_odk_validate=False, ) @@ -324,7 +326,7 @@ def test_xml__n_xpath_n_match_fail__xpath_count(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_count=self.suite2_counts + [(self.s2c3.xpath, 5)], + xml__xpath_count=[*self.suite2_counts, (self.s2c3.xpath, 5)], run_odk_validate=False, ) @@ -333,6 +335,6 @@ def test_xml__n_xpath_n_match_fail__xpath_match(self): with self.assertRaises(self.failureException): self.assertPyxformXform( md=self.md, - xml__xpath_match=self.suite2_xpaths + [self.s1c1.xpath], + xml__xpath_match=[*self.suite2_xpaths, self.s1c1.xpath], run_odk_validate=False, ) diff --git a/tests/test_pyxformtestcase.py b/tests/test_pyxformtestcase.py index e0b1acaa4..35a35a02b 100644 --- a/tests/test_pyxformtestcase.py +++ b/tests/test_pyxformtestcase.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Ensuring that the pyxform_test_case.PyxformTestCase class does some internal conversions correctly. diff --git a/tests/test_randomize_itemsets.py b/tests/test_randomize_itemsets.py index 6a5d90b03..6496e441e 100644 --- a/tests/test_randomize_itemsets.py +++ b/tests/test_randomize_itemsets.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test randomize itemsets. """ diff --git a/tests/test_range.py b/tests/test_range.py index 85850fa02..14bb5553e 100644 --- a/tests/test_range.py +++ b/tests/test_range.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test range widget. """ @@ -89,7 +88,7 @@ def test_range_type_float(self): ], ) - def test_range_type_invvalid_parameters(self): + def test_range_type_invalid_parameters(self): # 'increment' is an invalid property self.assertPyxformXform( name="data", @@ -168,7 +167,7 @@ def test_range_comma_separator(self): | survey | | | | | | | type | name | label | parameters | | | range | level | Scale | start = 1 , end = 10 , step = 2 | - """, # noqa + """, xml__contains=[ '', '', diff --git a/tests/test_rank.py b/tests/test_rank.py index 22d0001b8..f41386e4d 100644 --- a/tests/test_rank.py +++ b/tests/test_rank.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test rank widget. """ diff --git a/tests/test_repeat.py b/tests/test_repeat.py index 013ba2ff4..4bb0cb999 100644 --- a/tests/test_repeat.py +++ b/tests/test_repeat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test reapeat structure. """ @@ -64,7 +63,7 @@ def test_repeat_relative_reference(self): | | note | note9 | | ${FF} ${H} ${N} ${N} | | | end repeat | | | | | | | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long instance__contains=[ '
', "", @@ -119,7 +118,7 @@ def test_calculate_relative_path(self): | | crop_list | maize | Maize | | | | crop_list | beans | Beans | | | | crop_list | kale | Kale | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long model__contains=[ """""", @@ -149,10 +148,10 @@ def test_choice_filter_relative_path(self): # pylint: disable=invalid-name | | crop_list | maize | Maize | | | | crop_list | beans | Beans | | | | crop_list | kale | Kale | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""", # noqa pylint: disable=line-too-long - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -181,9 +180,9 @@ def test_indexed_repeat_relative_path(self): | | crop_list | maize | Maize | | | | crop_list | beans | Beans | | | | crop_list | kale | Kale | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long model__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -267,7 +266,7 @@ def test_hints_are_not_present_within_repeats(self): | | pet | cat | Cat | | | | pet | bird | Bird | | | | pet | fish | Fish | | - """ # noqa + """ self.assertPyxformXform( md=md, xml__xpath_match=[ @@ -294,7 +293,7 @@ def test_hints_are_present_within_groups(self): | | text | child_name | Name of child? | Should be a text | | | decimal | birthweight | Child birthweight (in kgs)? | Should be a decimal | | | end group | | | | - """ # noqa + """ expected = """ @@ -305,7 +304,7 @@ def test_hints_are_present_within_groups(self): Should be a decimal - """ # noqa + """ self.assertPyxformXform(md=md, xml__contains=[expected]) @@ -485,9 +484,9 @@ def test_indexed_repeat_regular_calculation_relative_path_exception(self): | | text | name | Enter name | | | | text | prev_name | Name in previous repeat instance | indexed-repeat(${name}, ${person}, ${pos}-1) | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long model__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -503,9 +502,9 @@ def test_indexed_repeat_dynamic_default_relative_path_exception(self): | | text | name | Enter name | | | | text | prev_name | Name in previous repeat instance | indexed-repeat(${name}, ${person}, position(..)-1) | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -524,9 +523,9 @@ def test_indexed_repeat_nested_repeat_relative_path_exception(self): | | text | prev_name | Non-sensible previous name in first family, 2nd person | indexed-repeat(${name}, ${family}, 1, ${person}, 2) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -548,9 +547,9 @@ def test_indexed_repeat_math_expression_nested_repeat_relative_path_exception( | | text | prev_name | Expression label | 7 * indexed-repeat(${age}, ${family}, 1, ${person}, 2) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -572,9 +571,9 @@ def test_multiple_indexed_repeat_in_expression_nested_repeat_relative_path_excep | | text | prev_name | Expression label | concat(indexed-repeat(${name}, ${family}, 1, ${person}, 2), indexed-repeat(${age}, ${family}, 1, ${person}, 2)) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -596,9 +595,9 @@ def test_mixed_variables_and_indexed_repeat_in_expression_text_type_nested_repea | | text | prev_name | Expression label | concat(${name}, indexed-repeat(${age}, ${family}, 1, ${person}, 2), ${age}) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -620,9 +619,9 @@ def test_mixed_variables_and_indexed_repeat_in_expression_integer_type_nested_re | | integer | bp_dia | Diastolic pressure | if(${bp_row} = 1, '', indexed-repeat(${bp_dia}, ${bp_rg}, ${bp_row} - 1)) | | | end repeat | | | | | | end group | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -644,9 +643,9 @@ def test_indexed_repeat_math_expression_with_double_variable_in_nested_repeat_re | | text | prev_name | Expression label | ${age} > indexed-repeat(${age}, ${family}, 1, ${person}, 2) | | | end repeat | | | | | | end repeat | | | | - """, # noqa pylint: disable=line-too-long + """, # pylint: disable=line-too-long xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -677,7 +676,7 @@ def test_repeat_using_select_with_reference_path_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -708,7 +707,7 @@ def test_repeat_using_select_uses_current_with_reference_path_in_predicate_and_i name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -734,7 +733,7 @@ def test_repeat_and_group_with_reference_path_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -758,7 +757,7 @@ def test_repeat_with_reference_path_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -784,9 +783,9 @@ def test_repeat_with_reference_path_with_spaces_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long - """""", # noqa pylint: disable=line-too-long - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long + """""", # pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -810,7 +809,7 @@ def test_repeat_with_reference_path_in_a_method_with_spaces_in_predicate_uses_cu name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -834,7 +833,7 @@ def test_repeat_with_reference_path_with_spaces_in_predicate_with_parenthesis_us name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -855,7 +854,7 @@ def test_relative_path_expansion_not_using_current_if_reference_path_is_predicat name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -878,7 +877,7 @@ def test_relative_path_expansion_not_using_current_if_reference_path_is_predicat name="data", md=xlsform_md, xml__contains=[ - """""" # noqa pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -902,7 +901,7 @@ def test_repeat_with_reference_path_in_multiple_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -928,7 +927,7 @@ def test_repeat_with_reference_path_in_multiple_complex_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -954,7 +953,7 @@ def test_repeat_with_reference_path_after_instance_in_predicate_uses_current( name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) @@ -980,7 +979,7 @@ def test_repeat_with_reference_path_after_instance_not_in_predicate_not_using_cu name="data", md=xlsform_md, xml__contains=[ - """""", # noqa pylint: disable=line-too-long + """""", # pylint: disable=line-too-long ], ) diff --git a/tests/test_repeat_template.py b/tests/test_repeat_template.py index 3f8ed79cc..155952162 100644 --- a/tests/test_repeat_template.py +++ b/tests/test_repeat_template.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test repeat template and instance structure. """ diff --git a/tests/test_secondary_instance_translations.py b/tests/test_secondary_instance_translations.py index 417e63ed3..bd80f709f 100644 --- a/tests/test_secondary_instance_translations.py +++ b/tests/test_secondary_instance_translations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Testing inlining translation when no translation is specified. """ diff --git a/tests/test_set_geopoint.py b/tests/test_set_geopoint.py index d897d063c..0540cbbfa 100644 --- a/tests/test_set_geopoint.py +++ b/tests/test_set_geopoint.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test setgeopoint widget. """ diff --git a/tests/test_settings_auto_send_delete.py b/tests/test_settings_auto_send_delete.py index ed88628f6..ac8df5af3 100644 --- a/tests/test_settings_auto_send_delete.py +++ b/tests/test_settings_auto_send_delete.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test settins auto settings. """ @@ -7,7 +6,6 @@ class SettingsAutoSendDelete(PyxformTestCase): def test_settings_auto_send_true(self): - self.assertPyxformXform( name="data", md=""" @@ -22,7 +20,6 @@ def test_settings_auto_send_true(self): ) def test_settings_auto_delete_true(self): - self.assertPyxformXform( name="data", md=""" @@ -37,7 +34,6 @@ def test_settings_auto_delete_true(self): ) def test_settings_auto_send_delete_false(self): - self.assertPyxformXform( name="data", md=""" @@ -52,7 +48,6 @@ def test_settings_auto_send_delete_false(self): ) def test_settings_without_submission_url_does_not_generate_method_attribute(self): - self.assertPyxformXform( name="data", md=""" @@ -75,7 +70,6 @@ def test_settings_without_submission_url_does_not_generate_method_attribute(self ) def test_settings_with_submission_url_generates_method_attribute(self): - self.assertPyxformXform( name="data", md=""" diff --git a/tests/settings_test.py b/tests/test_settings_xls.py similarity index 64% rename from tests/settings_test.py rename to tests/test_settings_xls.py index 85ebdf979..de52795ee 100644 --- a/tests/settings_test.py +++ b/tests/test_settings_xls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test settings sheet syntax. """ @@ -7,11 +6,11 @@ from pyxform.builder import create_survey_from_path from pyxform.xls2json import SurveyReader + from tests import utils class SettingsTests(TestCase): - maxDiff = None def setUp(self): @@ -20,26 +19,26 @@ def setUp(self): def test_survey_reader(self): survey_reader = SurveyReader(self.path, default_name="settings") expected_dict = { - u"id_string": u"new_id", - u"sms_keyword": u"new_id", - u"default_language": u"default", - u"name": u"settings", - u"title": u"My Survey", - u"type": u"survey", - u"attribute": { - u"my_number": u"1234567890", - u"my_string": u"lor\xe9m ipsum", + "id_string": "new_id", + "sms_keyword": "new_id", + "default_language": "default", + "name": "settings", + "title": "My Survey", + "type": "survey", + "attribute": { + "my_number": "1234567890", + "my_string": "lor\xe9m ipsum", }, - u"children": [ + "children": [ { - u"name": u"your_name", - u"label": {u"english": u"What is your name?"}, - u"type": u"text", + "name": "your_name", + "label": {"english": "What is your name?"}, + "type": "text", }, { - u"name": u"your_age", - u"label": {u"english": u"How many years old are you?"}, - u"type": u"integer", + "name": "your_age", + "label": {"english": "How many years old are you?"}, + "type": "integer", }, { "children": [ diff --git a/tests/test_sheet_columns.py b/tests/test_sheet_columns.py index 9b730b0f3..5b60c5498 100644 --- a/tests/test_sheet_columns.py +++ b/tests/test_sheet_columns.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XLSForm sheet names. """ diff --git a/tests/test_sms.py b/tests/test_sms.py index cf7e55b77..005780716 100644 --- a/tests/test_sms.py +++ b/tests/test_sms.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test sms syntax. """ diff --git a/tests/test_survey.py b/tests/test_survey.py index 6718d1143..cd2bec186 100644 --- a/tests/test_survey.py +++ b/tests/test_survey.py @@ -15,9 +15,7 @@ def test_many_xpath_references_do_not_hit_64_recursion_limit__one_to_one(self): | | text | q1 | Q1 | | | | note | n | {n} | | | | text | q2 | Q2 | {r} | - """.format( - n="q1 = ${q1} " * 250, r=" or ".join(["${q1} = 'y'"] * 250) - ), + """.format(n="q1 = ${q1} " * 250, r=" or ".join(["${q1} = 'y'"] * 250)), ) def test_many_xpath_references_do_not_hit_64_recursion_limit__many_to_one(self): @@ -31,8 +29,8 @@ def test_many_xpath_references_do_not_hit_64_recursion_limit__many_to_one(self): {q} | | note | n | {n} | """.format( - q="\n".join((tmpl_q.format(i) for i in range(1, 250))), - n=" ".join((tmpl_n.format(i) for i in range(1, 250))), + q="\n".join(tmpl_q.format(i) for i in range(1, 250)), + n=" ".join(tmpl_n.format(i) for i in range(1, 250)), ), ) @@ -47,7 +45,7 @@ def test_many_xpath_references_do_not_hit_64_recursion_limit__many_to_many(self) {q} {n} """.format( - q="\n".join((tmpl_q.format(i) for i in range(1, 250))), - n="\n".join((tmpl_n.format(i) for i in range(1, 250))), + q="\n".join(tmpl_q.format(i) for i in range(1, 250)), + n="\n".join(tmpl_n.format(i) for i in range(1, 250)), ), ) diff --git a/tests/test_table_list.py b/tests/test_table_list.py index a61409211..155c157c4 100644 --- a/tests/test_table_list.py +++ b/tests/test_table_list.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test table list appearance syntax. """ diff --git a/tests/test_translations.py b/tests/test_translations.py index 1346cf88f..7b3f2a561 100644 --- a/tests/test_translations.py +++ b/tests/test_translations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test translations syntax. """ @@ -7,13 +6,13 @@ from time import perf_counter from unittest.mock import patch -from pyxform.constants import CHOICES +from pyxform.constants import CHOICES, SURVEY from pyxform.constants import DEFAULT_LANGUAGE_VALUE as DEFAULT_LANG -from pyxform.constants import SURVEY from pyxform.validators.pyxform.translations_checks import ( OR_OTHER_WARNING, format_missing_translations_msg, ) + from tests.pyxform_test_case import PyxformTestCase from tests.xpath_helpers.choices import xpc from tests.xpath_helpers.questions import xpq @@ -323,8 +322,8 @@ def test_missing_translation__one_lang_all_cols(self): self.assertPyxformXform( md=md, warnings__contains=[warning], - xml__xpath_match=common_xpaths - + [ + xml__xpath_match=[ + *common_xpaths, xp.question_itext_label(DEFAULT_LANG, "hello"), xp.question_itext_hint(DEFAULT_LANG, "-"), xp.question_itext_form(DEFAULT_LANG, "guidance", "-"), @@ -361,8 +360,11 @@ def test_missing_translation__one_lang_all_cols(self): md=settings + md, # TODO: bug - missing default lang translatable/itext values. # warnings__contains=[warning], - xml__xpath_match=common_xpaths - + [xp.language_is_default("eng"), xp.language_no_itext(DEFAULT_LANG)], + xml__xpath_match=[ + *common_xpaths, + xp.language_is_default("eng"), + xp.language_no_itext(DEFAULT_LANG), + ], ) @unittest.skip("Slow performance test. Un-skip to run as needed.") @@ -394,27 +396,27 @@ def test_missing_translations_check_performance(self): | | c{i} | nb | lb-d | lb-e | """ for count in (500, 1000, 2000): - questions = "\n".join((question.format(i=i) for i in range(1, count))) - choice_lists = "\n".join((choice_list.format(i=i) for i in range(1, count))) + questions = "\n".join(question.format(i=i) for i in range(1, count)) + choice_lists = "\n".join(choice_list.format(i=i) for i in range(1, count)) md = "".join((survey_header, questions, choices_header, choice_lists)) - def run(name): + def run(name, case): runs = 0 results = [] while runs < 10: start = perf_counter() - self.assertPyxformXform(md=md) + self.assertPyxformXform(md=case) results.append(perf_counter() - start) runs += 1 print(name, sum(results) / len(results)) - run(name=f"questions={count}, with check (seconds):") + run(name=f"questions={count}, with check (seconds):", case=md) with patch( "pyxform.xls2json.SheetTranslations.missing_check", return_value=[], ): - run(name=f"questions={count}, without check (seconds):") + run(name=f"questions={count}, without check (seconds):", case=md) def test_translation_detection__survey_and_choices_columns_present(self): """Should identify that the survey is multi-language when first row(s) empty.""" diff --git a/tests/test_trigger.py b/tests/test_trigger.py index e8002a37d..b3cdb6727 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test handling setvalue of 'trigger' column in forms """ diff --git a/tests/tutorial_test.py b/tests/test_tutorial_xls.py similarity index 92% rename from tests/tutorial_test.py rename to tests/test_tutorial_xls.py index eb84cb395..57b15d537 100644 --- a/tests/tutorial_test.py +++ b/tests/test_tutorial_xls.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- """ Test tutorial XLSForm. """ from unittest import TestCase from pyxform.builder import create_survey_from_path + from tests import utils diff --git a/tests/test_typed_calculates.py b/tests/test_typed_calculates.py index 396afef80..5a13f3f77 100644 --- a/tests/test_typed_calculates.py +++ b/tests/test_typed_calculates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test that any row with a calculation becomes a calculate of the row's type or of type string if the type is "calculate". A hint or label error should only be thrown for a row without a calculation. diff --git a/tests/test_unicode_rtl.py b/tests/test_unicode_rtl.py index 130b6949b..ec28ad80a 100644 --- a/tests/test_unicode_rtl.py +++ b/tests/test_unicode_rtl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test unicode rtl in XLSForms. """ diff --git a/tests/test_upload_question.py b/tests/test_upload_question.py index c6abf6d6e..c611e940a 100644 --- a/tests/test_upload_question.py +++ b/tests/test_upload_question.py @@ -73,6 +73,7 @@ def test_image_question_custom_col_calc(self): | | text | watermark_phrase | Watermark Text: | | | | text | text1 | Text | | | | image | image1 | Take a Photo: | watermark=${watermark_phrase} | - """, # noqa + """, + errored=False, xml__contains=["watermark= /data/watermark_phrase "], ) diff --git a/tests/test_utils/md_table.py b/tests/test_utils/md_table.py index b1aafe17f..9903c417a 100644 --- a/tests/test_utils/md_table.py +++ b/tests/test_utils/md_table.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Markdown table utility functions. """ diff --git a/tests/test_validate_unicode_exception.py b/tests/test_validate_unicode_exception.py index d49fcde1a..4648f0cd4 100644 --- a/tests/test_validate_unicode_exception.py +++ b/tests/test_validate_unicode_exception.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test unicode characters in validate error messages. """ @@ -27,7 +26,7 @@ def test_validate_unicode_exception(self): def test_validate_with_more_unicode(self): self.assertPyxformXform( - md=u""" + md=""" | survey | | | | | | | type | name | label | calculation | | | calculate | bad | bad | £¥§©®₱₩ | diff --git a/tests/test_validator_update.py b/tests/test_validator_update.py index a311ec66b..9f7e3cb94 100644 --- a/tests/test_validator_update.py +++ b/tests/test_validator_update.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test validator update cli command. """ @@ -17,6 +16,7 @@ _UpdateInfo, capture_handler, ) + from tests import validators from tests.utils import get_temp_dir, get_temp_file from tests.validators.server import ThreadingServerInThread @@ -77,7 +77,6 @@ def test_get_temp_dir(self): class TestUpdateHandler(TestCase): - server: "Optional[ThreadingServerInThread]" = None @classmethod @@ -363,7 +362,7 @@ def test_unzip_find_zip_jobs__ok_real_current(self): open_zip_file=zip_file, bin_paths=bin_paths, out_path=temp_dir ) self.assertEqual(3, len(jobs.keys())) - self.assertTrue(list(jobs.keys())[0].startswith(temp_dir)) + self.assertTrue(next(iter(jobs.keys())).startswith(temp_dir)) def test_unzip_find_zip_jobs__ok_real_ideal(self): """Should return a list of zip jobs same length as search.""" @@ -377,7 +376,7 @@ def test_unzip_find_zip_jobs__ok_real_ideal(self): open_zip_file=zip_file, bin_paths=bin_paths, out_path=temp_dir ) self.assertEqual(3, len(jobs.keys())) - self.assertTrue(list(jobs.keys())[0].startswith(temp_dir)) + self.assertTrue(next(iter(jobs.keys())).startswith(temp_dir)) def test_unzip_find_zip_jobs__ok_real_dupes(self): """Should return a list of zip jobs same length as search.""" @@ -391,7 +390,7 @@ def test_unzip_find_zip_jobs__ok_real_dupes(self): open_zip_file=zip_file, bin_paths=bin_paths, out_path=temp_dir ) self.assertEqual(3, len(jobs.keys())) - self.assertTrue(list(jobs.keys())[0].startswith(temp_dir)) + self.assertTrue(next(iter(jobs.keys())).startswith(temp_dir)) def test_unzip_find_zip_jobs__not_found_raises(self): """Should raise an error if zip jobs isn't same length as search.""" @@ -420,9 +419,9 @@ def test_unzip_extract_file__bad_crc_raises(self): with get_temp_dir() as temp_dir, ZipFile( self.zip_file, mode="r" ) as zip_file, self.assertRaises(BadZipFile) as ctx: - zip_item = [ + zip_item = next( x for x in zip_file.infolist() if x.filename.endswith("validate") - ][0] + ) zip_item.CRC = 12345 file_out_path = os.path.join(temp_dir, "validate") self.updater._unzip_extract_file( diff --git a/tests/test_validator_util.py b/tests/test_validator_util.py index 42cbef3d1..147bf728a 100644 --- a/tests/test_validator_util.py +++ b/tests/test_validator_util.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test pyxform.validators.utils module. """ @@ -7,6 +6,7 @@ from pyxform.validators.error_cleaner import ErrorCleaner from pyxform.validators.util import XFORM_SPEC_PATH, check_readable + from tests.utils import prep_class_config diff --git a/tests/test_validators.py b/tests/test_validators.py index 39e2b48b0..0d5a08e09 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test validators. """ diff --git a/tests/test_warnings.py b/tests/test_warnings.py index f8b4ab693..2f8a38729 100644 --- a/tests/test_warnings.py +++ b/tests/test_warnings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test warnings. """ diff --git a/tests/test_whitespace.py b/tests/test_whitespace.py index 2f19615ac..9e3c9c01f 100644 --- a/tests/test_whitespace.py +++ b/tests/test_whitespace.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test whitespace around output variables in XForms. """ diff --git a/tests/test_xform2json.py b/tests/test_xform2json.py index 48efc0702..afc1d8d73 100644 --- a/tests/test_xform2json.py +++ b/tests/test_xform2json.py @@ -1,11 +1,106 @@ -# -*- coding: utf-8 -*- """ -Test XForm2JSON functionality +Test xform2json module. """ import json +import os +from unittest import TestCase +from xml.etree.ElementTree import ParseError -from pyxform.builder import create_survey_element_from_dict +from pyxform.builder import create_survey_element_from_dict, create_survey_from_path +from pyxform.xform2json import _try_parse, create_survey_element_from_xml + +from tests import test_output, utils from tests.pyxform_test_case import PyxformTestCase +from tests.xform_test_case.base import XFormTestCase + + +class DumpAndLoadXForm2JsonTests(XFormTestCase): + maxDiff = None + + def setUp(self): + self.excel_files = [ + "gps.xls", + # "include.xls", + "choice_filter_test.xlsx", + "specify_other.xls", + "loop.xls", + "text_and_integer.xls", + # todo: this is looking for json that is created (and + # deleted) by another test, is should just add that json + # to the directory. + # "include_json.xls", + "simple_loop.xls", + "yes_or_no_question.xls", + "xlsform_spec_test.xlsx", + "field-list.xlsx", + "table-list.xls", + "group.xls", + ] + self.surveys = {} + self.this_directory = os.path.dirname(__file__) + for filename in self.excel_files: + path = utils.path_to_text_fixture(filename) + self.surveys[filename] = create_survey_from_path(path) + + def test_load_from_dump(self): + for filename, survey in self.surveys.items(): + with self.subTest(msg=filename): + survey.json_dump() + survey_from_dump = create_survey_element_from_xml(survey.to_xml()) + expected = survey.to_xml() + observed = survey_from_dump.to_xml() + self.assertXFormEqual(expected, observed) + + def tearDown(self): + for survey in self.surveys.values(): + path = survey.name + ".json" + if os.path.exists(path): + os.remove(path) + + +class TestXMLParse(TestCase): + @classmethod + def setUpClass(cls): + cls.tidy_file = None + cls.xml = """\n1""" + + def tearDown(self): + if self.tidy_file is not None: + os.remove(self.tidy_file) + + def test_try_parse_with_string(self): + """Should return root node from XML string.""" + root = _try_parse(self.xml) + self.assertEqual("a", root.tag) + + def test_try_parse_with_path(self): + """Should return root node from XML file path.""" + xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") + self.tidy_file = xml_path + with open(xml_path, mode="w", encoding="utf-8") as xml_file: + xml_file.write(self.xml) + root = _try_parse(xml_path) + self.assertEqual("a", root.tag) + + def test_try_parse_with_bad_path(self): + """Should raise IOError: file doesn't exist.""" + xml_path = os.path.join(test_output.PATH, "not_a_real_file.xyz") + with self.assertRaises(IOError): + _try_parse(xml_path) + + def test_try_parse_with_bad_string(self): + """Should raise IOError: string parse failed and its not a path.""" + with self.assertRaises(IOError): + _try_parse("not valid xml :(") + + def test_try_parse_with_bad_file(self): + """Should raise XMLSyntaxError: file exists but content is not valid.""" + xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") + self.tidy_file = xml_path + with open(xml_path, mode="w", encoding="utf-8") as xml_file: + xml_file.write("not valid xml :(") + with self.assertRaises(ParseError): + _try_parse(xml_path) class TestXForm2JSON(PyxformTestCase): diff --git a/tests/test_xls2json.py b/tests/test_xls2json.py index 4fce54528..3d518b50b 100644 --- a/tests/test_xls2json.py +++ b/tests/test_xls2json.py @@ -1,9 +1,9 @@ import os import psutil - from pyxform.xls2json_backends import xlsx_to_dict from pyxform.xls2xform import get_xml_path, xls2xform_convert + from tests import example_xls, test_output from tests.pyxform_test_case import PyxformTestCase from tests.test_utils.md_table import md_table_to_workbook @@ -101,7 +101,7 @@ def test_workbook_to_json__ignore_prefixed_name__choices(self): md=CHOICES.format(name=n), errored=True, error__contains=[self.err_choices_required], - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__ignore_prefixed_name__external_choices(self): @@ -113,7 +113,7 @@ def test_workbook_to_json__ignore_prefixed_name__external_choices(self): md=EXTERNAL_CHOICES.format(name=n), errored=True, error__contains=[self.err_ext_choices_required], - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__ignore_prefixed_name__settings(self): @@ -135,7 +135,7 @@ def test_workbook_to_json__ignore_prefixed_name__survey(self): md=SURVEY.format(name=n), errored=True, error__contains=[self.err_survey_required], - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_found__choices(self): @@ -149,7 +149,7 @@ def test_workbook_to_json__misspelled_found__choices(self): error__contains=[ self.err_choices_required, self.err_similar_found, - "'{}'".format(n), + f"'{n}'", ], ) @@ -206,7 +206,7 @@ def test_workbook_to_json__misspelled_found__external_choices(self): error__contains=[ self.err_ext_choices_required, self.err_similar_found, - "'{}'".format(n), + f"'{n}'", ], ) @@ -265,7 +265,7 @@ def test_workbook_to_json__misspelled_found__settings(self): self.assertPyxformXform( name="test", md=SETTINGS.format(name=n), - warnings__contains=[self.err_similar_found, "'{}'".format(n)], + warnings__contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_found__settings_exists(self): @@ -315,7 +315,7 @@ def test_workbook_to_json__misspelled_found__survey(self): error__contains=[ self.err_survey_required, self.err_similar_found, - "'{}'".format(n), + f"'{n}'", ], ) @@ -363,7 +363,7 @@ def test_workbook_to_json__misspelled_not_found__choices(self): name="test", md=CHOICES.format(name=n), errored=True, - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_not_found__external_choices(self): @@ -374,7 +374,7 @@ def test_workbook_to_json__misspelled_not_found__external_choices(self): name="test", md=EXTERNAL_CHOICES.format(name=n), errored=True, - error__not_contains=[self.err_similar_found, "'{}'".format(n)], + error__not_contains=[self.err_similar_found, f"'{n}'"], ) def test_workbook_to_json__misspelled_not_found__settings(self): diff --git a/tests/test_xls2json_backends.py b/tests/test_xls2json_backends.py index 50285c3b6..55a9fcfcf 100644 --- a/tests/test_xls2json_backends.py +++ b/tests/test_xls2json_backends.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xls2json_backends module functionality. """ @@ -8,13 +7,13 @@ import openpyxl import xlrd - from pyxform.xls2json_backends import ( xls_to_dict, xls_value_to_unicode, xlsx_to_dict, xlsx_value_to_str, ) + from tests import bug_example_xls, utils diff --git a/tests/xls2json_tests.py b/tests/test_xls2json_xls.py similarity index 90% rename from tests/xls2json_tests.py rename to tests/test_xls2json_xls.py index 6e5360aa5..1f914968a 100644 --- a/tests/xls2json_tests.py +++ b/tests/test_xls2json_xls.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- """ Testing simple cases for Xls2Json """ -import codecs import json import os from unittest import TestCase from pyxform.xls2json import SurveyReader from pyxform.xls2json_backends import csv_to_dict, xls_to_dict, xlsx_to_dict + from tests import example_xls, test_expected_output, test_output, utils @@ -32,14 +31,13 @@ def test_simple_yes_or_no_question(self): ) x = SurveyReader(path_to_excel_file, default_name="yes_or_no_question") x_results = x.to_json_dict() - with codecs.open(output_path, mode="w", encoding="utf-8") as fp: + with open(output_path, mode="w", encoding="utf-8") as fp: json.dump(x_results, fp=fp, ensure_ascii=False, indent=4) # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: - expected_json = json.load(expected_file) - actual_json = json.load(actual_file) - self.assertEqual(expected_json, actual_json) + with open(expected_output_path, encoding="utf-8") as expected, open( + output_path, encoding="utf-8" + ) as observed: + self.assertEqual(json.load(expected), json.load(observed)) def test_hidden(self): x = SurveyReader(utils.path_to_text_fixture("hidden.xls"), default_name="hidden") @@ -127,14 +125,13 @@ def test_table(self): ) x = SurveyReader(path_to_excel_file, default_name="simple_loop") x_results = x.to_json_dict() - with codecs.open(output_path, mode="w", encoding="utf-8") as fp: + with open(output_path, mode="w", encoding="utf-8") as fp: json.dump(x_results, fp=fp, ensure_ascii=False, indent=4) # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: - expected_json = json.load(expected_file) - actual_json = json.load(actual_file) - self.assertEqual(expected_json, actual_json) + with open(expected_output_path, encoding="utf-8") as expected, open( + output_path, encoding="utf-8" + ) as observed: + self.assertEqual(json.load(expected), json.load(observed)) def test_choice_filter_choice_fields(self): """ diff --git a/tests/xls2xform_tests.py b/tests/test_xls2xform.py similarity index 94% rename from tests/xls2xform_tests.py rename to tests/test_xls2xform.py index 5d9b8898b..7329d2738 100644 --- a/tests/xls2xform_tests.py +++ b/tests/test_xls2xform.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test xls2xform module. """ @@ -8,38 +7,19 @@ import argparse import logging -from unittest import TestCase +from unittest import TestCase, mock -import pyxform from pyxform.xls2xform import ( _create_parser, _validator_args_logic, get_xml_path, main_cli, ) -from tests.utils import path_to_text_fixture -try: - from unittest import mock -except ImportError: - import mock +from tests.utils import path_to_text_fixture class XLS2XFormTests(TestCase): - survey_package = { - "id_string": "test_2011_08_29b", - "name_of_main_section": "gps", - "sections": { - "gps": { - "children": [{"name": "location", "type": "gps"}], - "name": "gps", - "type": "survey", - } - }, - "title": "test", - } - survey = pyxform.create_survey(**survey_package) - def test_create_parser_without_args(self): """Should exit when no args provided.""" with self.assertRaises(SystemExit): diff --git a/tests/test_xlsform_headers.py b/tests/test_xlsform_headers.py index c588b72a0..b1fa34222 100644 --- a/tests/test_xlsform_headers.py +++ b/tests/test_xlsform_headers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XLSForm headers syntax. """ diff --git a/tests/test_xlsform_spec.py b/tests/test_xlsform_spec.py index 42e72f281..4f76f2880 100644 --- a/tests/test_xlsform_spec.py +++ b/tests/test_xlsform_spec.py @@ -53,7 +53,7 @@ def test_warnings__count(self): | settings | | | | | | | | form_title | form_id | public_key | submission_url | default_language | | | spec_test | spec_test | | | | - """ # noqa + """ warnings = [] self.assertPyxformXform( debug=True, diff --git a/tests/test_xml_structure.py b/tests/test_xml_structure.py index 120a6e2a2..6be5e6442 100644 --- a/tests/test_xml_structure.py +++ b/tests/test_xml_structure.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XForm structure. """ diff --git a/tests/utils.py b/tests/utils.py index e4dbd0771..b2b2344d4 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ The tests utils module functionality. """ @@ -12,6 +11,7 @@ from pyxform import file_utils from pyxform.builder import create_survey, create_survey_from_path + from tests import example_xls if TYPE_CHECKING: @@ -28,7 +28,7 @@ def build_survey(filename): def create_survey_from_fixture(fixture_name, filetype="xls", include_directory=False): - fixture_path = path_to_text_fixture("%s.%s" % (fixture_name, filetype)) + fixture_path = path_to_text_fixture(f"{fixture_name}.{filetype}") noop, section_dict = file_utils.load_file_to_dict(fixture_path) pkg = {"main_section": section_dict} if include_directory: @@ -93,11 +93,10 @@ def truncate_temp_files(temp_dir): for f in os.scandir(temp_dir): if os.path.isdir(f.path): truncate_temp_files(f.path) - else: - # Check still in temp directory - if f.path.startswith(temp_root): - with open(f.path, mode="w") as _: - pass + # Check still in temp directory + elif f.path.startswith(temp_root): + with open(f.path, mode="w", encoding="utf-8") as _: + pass def cleanup_pyxform_temp_files(prefix: str): diff --git a/tests/validators/__init__.py b/tests/validators/__init__.py index abbc7fd26..1b011eb96 100644 --- a/tests/validators/__init__.py +++ b/tests/validators/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os HERE = os.path.dirname(__file__) diff --git a/tests/validators/server.py b/tests/validators/server.py index 8acb9ee78..52d579dd3 100644 --- a/tests/validators/server.py +++ b/tests/validators/server.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os import posixpath import threading @@ -10,7 +9,7 @@ HERE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") -class SimpleHTTPRequestHandlerHere(SimpleHTTPRequestHandler, object): +class SimpleHTTPRequestHandlerHere(SimpleHTTPRequestHandler): def send_head(self): if self.client_address[0] != "127.0.0.1": self.send_error( @@ -18,7 +17,7 @@ def send_head(self): ) return None else: - return super(SimpleHTTPRequestHandlerHere, self).send_head() + return super().send_head() def translate_path(self, path): """ @@ -76,10 +75,10 @@ def _bind_and_activate(self, tries: int = 0): try: self.httpd.server_bind() self.httpd.server_activate() - except OSError as e: + except OSError: self.httpd.server_close() if 5 < tries: - raise e + raise else: time.sleep(0.5) self._bind_and_activate(tries=tries) diff --git a/tests/xform2json_test.py b/tests/xform2json_test.py deleted file mode 100644 index 31f6a6702..000000000 --- a/tests/xform2json_test.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Test xform2json module. -""" -import os -from unittest import TestCase -from xml.etree.ElementTree import ParseError - -from pyxform.builder import create_survey_from_path -from pyxform.xform2json import _try_parse, create_survey_element_from_xml -from tests import test_output, utils -from tests.pyxform_test_case import PyxformTestCase -from tests.xform_test_case.base import XFormTestCase - - -class DumpAndLoadXForm2JsonTests(XFormTestCase, PyxformTestCase): - - maxDiff = None - - def setUp(self): - self.excel_files = [ - "gps.xls", - # "include.xls", - "choice_filter_test.xlsx", - "specify_other.xls", - "loop.xls", - "text_and_integer.xls", - # todo: this is looking for json that is created (and - # deleted) by another test, is should just add that json - # to the directory. - # "include_json.xls", - "simple_loop.xls", - "yes_or_no_question.xls", - "xlsform_spec_test.xlsx", - "field-list.xlsx", - "table-list.xls", - "group.xls", - ] - self.surveys = {} - self.this_directory = os.path.dirname(__file__) - for filename in self.excel_files: - path = utils.path_to_text_fixture(filename) - self.surveys[filename] = create_survey_from_path(path) - - def test_load_from_dump(self): - for filename, survey in iter(self.surveys.items()): - with self.subTest(msg=filename): - survey.json_dump() - survey_from_dump = create_survey_element_from_xml(survey.to_xml()) - expected = survey.to_xml() - observed = survey_from_dump.to_xml() - self.assertXFormEqual(expected, observed) - - def tearDown(self): - for filename, survey in self.surveys.items(): - path = survey.name + ".json" - if os.path.exists(path): - os.remove(path) - - -class TestXMLParse(TestCase): - @classmethod - def setUpClass(cls): - cls.tidy_file = None - cls.xml = """\n1""" - - def tearDown(self): - if self.tidy_file is not None: - os.remove(self.tidy_file) - - def test_try_parse_with_string(self): - """Should return root node from XML string.""" - root = _try_parse(self.xml) - self.assertEqual("a", root.tag) - - def test_try_parse_with_path(self): - """Should return root node from XML file path.""" - xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") - self.tidy_file = xml_path - with open(xml_path, "w") as xml_file: - xml_file.write(self.xml) - root = _try_parse(xml_path) - self.assertEqual("a", root.tag) - - def test_try_parse_with_bad_path(self): - """Should raise IOError: file doesn't exist.""" - xml_path = os.path.join(test_output.PATH, "not_a_real_file.xyz") - with self.assertRaises(IOError): - _try_parse(xml_path) - - def test_try_parse_with_bad_string(self): - """Should raise IOError: string parse failed and its not a path.""" - with self.assertRaises(IOError): - _try_parse("not valid xml :(") - - def test_try_parse_with_bad_file(self): - """Should raise XMLSyntaxError: file exists but content is not valid.""" - xml_path = os.path.join(test_output.PATH, "test_try_parse.xml") - self.tidy_file = xml_path - with open(xml_path, "w") as xml_file: - xml_file.write("not valid xml :(") - with self.assertRaises(ParseError): - _try_parse(xml_path) diff --git a/tests/xform_test_case/base.py b/tests/xform_test_case/base.py index 791c023ae..b99dd0314 100644 --- a/tests/xform_test_case/base.py +++ b/tests/xform_test_case/base.py @@ -1,7 +1,7 @@ import os -import xml.etree.ElementTree as ETree from unittest import TestCase +from defusedxml.ElementTree import fromstring from formencode.doctest_xml_compare import xml_compare from tests import example_xls, test_output @@ -28,8 +28,8 @@ def get_file_path(self, filename): self.output_path = os.path.join(test_output.PATH, self.root_filename + ".xml") def assertXFormEqual(self, xform1, xform2): - xform1 = ETree.fromstring(xform1.encode("utf-8")) - xform2 = ETree.fromstring(xform2.encode("utf-8")) + xform1 = fromstring(xform1.encode("utf-8")) + xform2 = fromstring(xform2.encode("utf-8")) # Sort tags under section in each form self.sort_model(xform1) diff --git a/tests/xform_test_case/attributecolumnstest.py b/tests/xform_test_case/test_attribute_columns.py similarity index 70% rename from tests/xform_test_case/attributecolumnstest.py rename to tests/xform_test_case/test_attribute_columns.py index 1ad6cfe23..e560ff3be 100644 --- a/tests/xform_test_case/attributecolumnstest.py +++ b/tests/xform_test_case/test_attribute_columns.py @@ -1,21 +1,18 @@ -# -*- coding: utf-8 -*- """ Some tests for the new (v0.9) spec is properly implemented. """ -import codecs import os -import unittest import pyxform + from tests import test_expected_output from tests.xform_test_case.base import XFormTestCase class AttributeColumnsTest(XFormTestCase): - maxDiff = None - def runTest(self): + def test_conversion(self): filename = "attribute_columns_test.xlsx" self.get_file_path(filename) expected_output_path = os.path.join( @@ -33,10 +30,7 @@ def runTest(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) - - -if __name__ == "__main__": - unittest.main() + with open(expected_output_path, encoding="utf-8") as expected, open( + self.output_path, encoding="utf-8" + ) as observed: + self.assertXFormEqual(expected.read(), observed.read()) diff --git a/tests/xform_test_case/bug_tests.py b/tests/xform_test_case/test_bugs.py similarity index 82% rename from tests/xform_test_case/bug_tests.py rename to tests/xform_test_case/test_bugs.py index 08bca787c..df627593b 100644 --- a/tests/xform_test_case/bug_tests.py +++ b/tests/xform_test_case/test_bugs.py @@ -1,24 +1,24 @@ -# -*- coding: utf-8 -*- """ Some tests for the new (v0.9) spec is properly implemented. """ -import codecs import os -import unittest +from unittest import TestCase import pyxform +from pyxform.errors import PyXFormError from pyxform.utils import has_external_choices from pyxform.validators.odk_validate import ODKValidateError, check_xform from pyxform.xls2json import SurveyReader, parse_file_to_workbook_dict from pyxform.xls2json_backends import xlsx_to_dict + from tests import bug_example_xls, example_xls, test_expected_output, test_output from tests.xform_test_case.base import XFormTestCase -class GroupNames(unittest.TestCase): +class GroupNames(TestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "group_name_test.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -26,7 +26,7 @@ def runTest(self): output_path = os.path.join(test_output.PATH, root_filename + ".xml") # Do the conversion: warnings = [] - with self.assertRaises(Exception): + with self.assertRaises(PyXFormError): json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, default_name="group_name_test", warnings=warnings ) @@ -34,10 +34,10 @@ def runTest(self): survey.print_xform_to_file(output_path, warnings=warnings) -class NotClosedGroup(unittest.TestCase): +class NotClosedGroup(TestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "not_closed_group_test.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -45,7 +45,7 @@ def runTest(self): output_path = os.path.join(test_output.PATH, root_filename + ".xml") # Do the conversion: warnings = [] - with self.assertRaises(Exception): + with self.assertRaises(PyXFormError): json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, default_name="not_closed_group_test", @@ -55,10 +55,10 @@ def runTest(self): survey.print_xform_to_file(output_path, warnings=warnings) -class DuplicateColumns(unittest.TestCase): +class DuplicateColumns(TestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "duplicate_columns.xlsx" path_to_excel_file = os.path.join(example_xls.PATH, filename) # Get the xform output path: @@ -66,7 +66,7 @@ def runTest(self): output_path = os.path.join(test_output.PATH, root_filename + ".xml") # Do the conversion: warnings = [] - with self.assertRaises(Exception): + with self.assertRaises(PyXFormError): json_survey = pyxform.xls2json.parse_file_to_json( path_to_excel_file, default_name="duplicate_columns", warnings=warnings ) @@ -77,7 +77,7 @@ def runTest(self): class RepeatDateTest(XFormTestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "repeat_date_test.xls" self.get_file_path(filename) expected_output_path = os.path.join( @@ -93,15 +93,16 @@ def runTest(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) + with open(expected_output_path, encoding="utf-8") as expected, open( + self.output_path, encoding="utf-8" + ) as observed: + self.assertXFormEqual(expected.read(), observed.read()) class XmlEscaping(XFormTestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "xml_escaping.xls" self.get_file_path(filename) expected_output_path = os.path.join( @@ -117,15 +118,16 @@ def runTest(self): survey.print_xform_to_file(self.output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(self.output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) + with open(expected_output_path, encoding="utf-8") as expected, open( + self.output_path, encoding="utf-8" + ) as observed: + self.assertXFormEqual(expected.read(), observed.read()) class DefaultTimeTest(XFormTestCase): maxDiff = None - def runTest(self): + def test_conversion(self): filename = "default_time_demo.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -143,16 +145,17 @@ def runTest(self): survey.print_xform_to_file(output_path, warnings=warnings) # print warnings # Compare with the expected output: - with codecs.open(expected_output_path, "rb", encoding="utf-8") as expected_file: - with codecs.open(output_path, "rb", encoding="utf-8") as actual_file: - self.assertXFormEqual(expected_file.read(), actual_file.read()) + with open(expected_output_path, encoding="utf-8") as expected, open( + output_path, encoding="utf-8" + ) as observed: + self.assertXFormEqual(expected.read(), observed.read()) -class ValidateWrapper(unittest.TestCase): +class ValidateWrapper(TestCase): maxDiff = None @staticmethod - def runTest(): + def test_conversion(): filename = "ODKValidateWarnings.xlsx" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) # Get the xform output path: @@ -167,8 +170,8 @@ def runTest(): survey.print_xform_to_file(output_path, warnings=warnings) -class EmptyStringOnRelevantColumnTest(unittest.TestCase): - def runTest(self): +class EmptyStringOnRelevantColumnTest(TestCase): + def test_conversion(self): filename = "ict_survey_fails.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) workbook_dict = pyxform.xls2json.parse_file_to_workbook_dict(path_to_excel_file) @@ -177,8 +180,8 @@ def runTest(self): workbook_dict["survey"][0]["bind: relevant"].strip() -class BadChoicesSheetHeaders(unittest.TestCase): - def runTest(self): +class BadChoicesSheetHeaders(TestCase): + def test_conversion(self): filename = "spaces_in_choices_header.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) warnings = [] @@ -187,7 +190,10 @@ def runTest(self): default_name="spaces_in_choices_header", warnings=warnings, ) - self.assertEquals(len(warnings), 3, "Found " + str(len(warnings)) + " warnings") + # The "column with no header" warning is probably not reachable since XLS/X + # pre-processing ignores any columns without a header. + observed = [w for w in warnings if "Headers cannot include spaces" in w] + self.assertEqual(1, len(observed), warnings) def test_values_with_spaces_are_cleaned(self): """ @@ -208,7 +214,7 @@ def test_values_with_spaces_are_cleaned(self): ) -class TestChoiceNameAsType(unittest.TestCase): +class TestChoiceNameAsType(TestCase): def test_choice_name_as_type(self): filename = "choice_name_as_type.xls" path_to_excel_file = os.path.join(example_xls.PATH, filename) @@ -217,7 +223,7 @@ def test_choice_name_as_type(self): self.assertTrue(has_external_choices(survey_dict)) -class TestBlankSecondRow(unittest.TestCase): +class TestBlankSecondRow(TestCase): def test_blank_second_row(self): filename = "blank_second_row.xls" path_to_excel_file = os.path.join(bug_example_xls.PATH, filename) @@ -226,7 +232,7 @@ def test_blank_second_row(self): self.assertTrue(len(survey_dict) > 0) -class TestXLDateAmbigous(unittest.TestCase): +class TestXLDateAmbigous(TestCase): """Test non standard sheet with exception is processed successfully.""" def test_xl_date_ambigous(self): @@ -238,7 +244,7 @@ def test_xl_date_ambigous(self): self.assertTrue(len(survey_dict) > 0) -class TestXLDateAmbigousNoException(unittest.TestCase): +class TestXLDateAmbigousNoException(TestCase): """Test date values that exceed the workbook datemode value. (This would cause an exception with xlrd, but openpyxl handles it).""" @@ -250,7 +256,7 @@ def test_xl_date_ambigous_no_exception(self): self.assertEqual(survey_dict["survey"][4]["default"], "1900-01-01 00:00:00") -class TestSpreadSheetFilesWithMacrosAreAllowed(unittest.TestCase): +class TestSpreadSheetFilesWithMacrosAreAllowed(TestCase): """Test that spreadsheets with .xlsm extension are allowed""" def test_xlsm_files_are_allowed(self): @@ -260,14 +266,10 @@ def test_xlsm_files_are_allowed(self): self.assertIsInstance(result, dict) -class TestBadCalculation(unittest.TestCase): +class TestBadCalculation(TestCase): """Bad calculation should not kill the application""" def test_bad_calculate_javarosa_error(self): filename = "bad_calc.xml" test_xml = os.path.join(test_output.PATH, filename) self.assertRaises(ODKValidateError, check_xform, test_xml) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/xform_test_case/xlsform_spec_test.py b/tests/xform_test_case/test_xlsform_spec.py similarity index 92% rename from tests/xform_test_case/xlsform_spec_test.py rename to tests/xform_test_case/test_xlsform_spec.py index 61d08b647..80c5ba5c2 100644 --- a/tests/xform_test_case/xlsform_spec_test.py +++ b/tests/xform_test_case/test_xlsform_spec.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Some tests for the new (v0.9) spec is properly implemented. """ @@ -7,6 +6,7 @@ from pyxform import builder, xls2json from pyxform.errors import PyXFormError + from tests import example_xls, test_expected_output from tests.xform_test_case.base import XFormTestCase @@ -35,8 +35,8 @@ def compare_xform(self, file_name: str, set_default_name: bool = True): survey = builder.create_survey_element_from_dict(json_survey) survey.print_xform_to_file(self.output_path, warnings=warnings) # Compare with the expected output: - with open(expected_output_path, "r", encoding="utf-8") as ef, open( - self.output_path, "r", encoding="utf-8" + with open(expected_output_path, encoding="utf-8") as ef, open( + self.output_path, encoding="utf-8" ) as af: expected = ef.read() observed = af.read() @@ -66,7 +66,7 @@ class TestCalculateWithoutCalculation(TestCase): Just checks that calculate field without calculation raises a PyXFormError. """ - def runTest(self): + def test_conversion(self): filename = "calculate_without_calculation.xls" path_to_excel_file = os.path.join(example_xls.PATH, filename) self.assertRaises(PyXFormError, xls2json.parse_file_to_json, path_to_excel_file) diff --git a/tests/xform_test_case/xml_tests.py b/tests/xform_test_case/test_xml.py similarity index 99% rename from tests/xform_test_case/xml_tests.py rename to tests/xform_test_case/test_xml.py index 5908aa861..cfeb4271b 100644 --- a/tests/xform_test_case/xml_tests.py +++ b/tests/xform_test_case/test_xml.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test XForm XML syntax. """ @@ -7,6 +6,7 @@ from pyxform import create_survey_from_xls from pyxform.utils import node + from tests.utils import path_to_text_fixture from tests.xform_test_case.base import XFormTestCase diff --git a/tests/xpath_helpers/choices.py b/tests/xpath_helpers/choices.py index d546221b9..4489199ca 100644 --- a/tests/xpath_helpers/choices.py +++ b/tests/xpath_helpers/choices.py @@ -22,7 +22,7 @@ def model_instance_choices_label(cname: str, choices: Tuple[Tuple[str, str], ... for cv, cl in choices ) ) - return fr""" + return rf""" /h:html/h:head/x:model/x:instance[@id='{cname}']/x:root[ {choices_xp} ] @@ -41,7 +41,7 @@ def model_instance_choices_itext(cname: str, choices: Tuple[str, ...]): for idx, cv in enumerate(choices) ) ) - return fr""" + return rf""" /h:html/h:head/x:model/x:instance[@id='{cname}']/x:root[ {choices_xp} ] diff --git a/tests/xpath_helpers/questions.py b/tests/xpath_helpers/questions.py index bf865ab35..95d31ba26 100644 --- a/tests/xpath_helpers/questions.py +++ b/tests/xpath_helpers/questions.py @@ -6,14 +6,14 @@ class XPathHelper: @staticmethod def model_instance_item(q_name: str): """Model instance contains the question item.""" - return fr""" + return rf""" /h:html/h:head/x:model/x:instance/x:test_name/x:{q_name} """ @staticmethod def model_instance_bind(q_name: str, _type: str): """Model instance contains the question item.""" - return fr""" + return rf""" /h:html/h:head/x:model/x:bind[ @nodeset='/test_name/{q_name}' and @type='{_type}' @@ -64,7 +64,7 @@ def body_label_itext(q_type: str, q_name: str): @staticmethod def body_select1_itemset(q_name: str): """Body has a select1 with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/x:select1[ @ref = '/test_name/{q_name}' and ./x:itemset @@ -75,7 +75,7 @@ def body_select1_itemset(q_name: str): @staticmethod def body_group_select1_itemset(g_name: str, q_name: str): """Body has a select1 with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/x:group[@ref='/test_name/{g_name}']/x:select1[ @ref = '/test_name/{g_name}/{q_name}' and ./x:itemset @@ -86,7 +86,7 @@ def body_group_select1_itemset(g_name: str, q_name: str): @staticmethod def body_repeat_select1_itemset(r_name: str, q_name: str): """Body has a select1 with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/x:group[@ref='/test_name/{r_name}'] /x:repeat[@nodeset='/test_name/{r_name}'] /x:select1[ @@ -99,7 +99,7 @@ def body_repeat_select1_itemset(r_name: str, q_name: str): @staticmethod def body_odk_rank_itemset(q_name: str): """Body has a rank with an itemset, and no inline items.""" - return fr""" + return rf""" /h:html/h:body/odk:rank[ @ref = '/test_name/{q_name}' and ./x:itemset @@ -110,7 +110,7 @@ def body_odk_rank_itemset(q_name: str): @staticmethod def body_input_label_output_value(q_name: str): """Body has an input (note) with output reference in the label.""" - return fr""" + return rf""" /h:html/h:body/x:input[@ref='/test_name/{q_name}']/x:label/x:output/@value """