From e4dd353e9699a1396bbe9f7d288149f653318487 Mon Sep 17 00:00:00 2001 From: Eric Hunsberger Date: Wed, 31 Jul 2019 21:49:43 -0400 Subject: [PATCH] Initial version of rng and seed fixtures This is similar to what we had in Nengo core, except: - We use a `salt` string instead of a `seed-offset` integer. Strings are easier to work with than integers. - We now have a full test suite. It uses pytest's cache fixture, so when running the fixture tests manually, pytest has to be run twice. Normally tests would be run with test_pytest.py, which runs the test suite twice. Co-authored-by: Trevor Bekolay --- .codecov.yml | 19 ++++++ .gitignore | 26 +++++++ .gitlint | 12 ++++ .nengobones.yml | 78 +++++++++++++++++++++ .pre-commit-config.yaml | 7 ++ .templates/setup.cfg.template | 7 ++ .travis.yml | 97 ++++++++++++++++++++++++++ CHANGES.rst | 26 +++++++ CONTRIBUTING.rst | 46 +++++++++++++ CONTRIBUTORS.rst | 9 +++ LICENSE.rst | 29 ++++++++ MANIFEST.in | 37 ++++++++++ README.rst | 87 +++++++++++++++++++++++ docs/_static/favicon.ico | Bin 0 -> 15086 bytes docs/_templates/sidebar.html | 55 +++++++++++++++ docs/conf.py | 73 ++++++++++++++++++++ docs/index.rst | 9 +++ pyproject.toml | 7 ++ pytest_rng/__init__.py | 6 ++ pytest_rng/plugin.py | 59 ++++++++++++++++ pytest_rng/tests/__init__.py | 0 pytest_rng/tests/test_fixtures.py | 70 +++++++++++++++++++ pytest_rng/tests/test_pytest.py | 110 ++++++++++++++++++++++++++++++ pytest_rng/version.py | 16 +++++ setup.cfg | 109 +++++++++++++++++++++++++++++ setup.py | 67 ++++++++++++++++++ 26 files changed, 1061 insertions(+) create mode 100644 .codecov.yml create mode 100644 .gitignore create mode 100644 .gitlint create mode 100644 .nengobones.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .templates/setup.cfg.template create mode 100644 .travis.yml create mode 100644 CHANGES.rst create mode 100644 CONTRIBUTING.rst create mode 100644 CONTRIBUTORS.rst create mode 100644 LICENSE.rst create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 docs/_static/favicon.ico create mode 100644 docs/_templates/sidebar.html create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 pyproject.toml create mode 100644 pytest_rng/__init__.py create mode 100644 pytest_rng/plugin.py create mode 100644 pytest_rng/tests/__init__.py create mode 100644 pytest_rng/tests/test_fixtures.py create mode 100644 pytest_rng/tests/test_pytest.py create mode 100644 pytest_rng/version.py create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..0d7f5b1 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,19 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +codecov: + ci: + - "!ci.appveyor.com" + notify: + require_ci_to_pass: no + +coverage: + status: + project: + default: + enabled: yes + target: auto + patch: + default: + enabled: yes + target: 100% + changes: no diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24c1162 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +*.py[co] +.DS_Store +_build +build +dist +*.egg-info +*~ +*.bak +*.swp +log.txt +.ipynb_checkpoints/ +.cache +.tox +.vagrant +wintest.sh +Vagrantfile +*.class +*.eggs/ +.coverage +htmlcov +*.dist-info/ +.vscode +.idea +.pytest_cache/ + +.ci/*.sh diff --git a/.gitlint b/.gitlint new file mode 100644 index 0000000..054991b --- /dev/null +++ b/.gitlint @@ -0,0 +1,12 @@ +[general] +ignore=body-is-missing + +[title-max-length] +line-length=50 + +[B1] +# body line length +line-length=72 + +[title-match-regex] +regex=^[A-Z] diff --git a/.nengobones.yml b/.nengobones.yml new file mode 100644 index 0000000..52a527f --- /dev/null +++ b/.nengobones.yml @@ -0,0 +1,78 @@ +project_name: pytest-rng +pkg_name: pytest_rng +repo_name: nengo/pytest-rng +description: Fixtures for seeding tests and making randomness reproducible + +copyright_start: 2019 + +license_rst: + type: mit + +contributing_rst: {} + +contributors_rst: {} + +manifest_in: {} + +setup_py: + license: MIT license + python_requires: ">=3.5" + install_req: + - numpy + - pytest + docs_req: + - nengo_sphinx_theme>=1.0 + - sphinx + entry_points: + pytest11: + - "rng = pytest_rng.plugin" + classifiers: + - "Development Status :: 5 - Production/Stable" + - "Framework :: Pytest" + - "License :: OSI Approved :: MIT License" + - "Programming Language :: Python :: 3 :: Only" + - "Programming Language :: Python :: 3.5" + - "Programming Language :: Python :: 3.6" + - "Programming Language :: Python :: 3.7" + +setup_cfg: + pytest: + addopts: [] + filterwarnings: + - ignore:testdir.copy_example is an experimental api + rng_salt: v1.0.0 + pytester_example_dir: pytest_rng/tests + python_files: test_pytest.py + pylint: + disable: + - missing-docstring + +docs_conf_py: + nengo_logo: general-small-light.svg + +travis_yml: + python: 3.6 + jobs: + - script: static + - script: test-coverage + - script: test + python: 3.5 + cache: false # disable the cache for one build to make sure that works + - script: test + python: 3.7 + dist: xenial # currently only xenial has python 3.7 + - script: docs + +ci_scripts: + - template: static + - template: test + - template: test + output_name: test-coverage + coverage: true + - template: docs + +codecov_yml: {} + +pre_commit_config_yaml: {} + +pyproject_toml: {} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6698fe1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +repos: +- repo: https://github.com/psf/black + rev: stable + hooks: + - id: black diff --git a/.templates/setup.cfg.template b/.templates/setup.cfg.template new file mode 100644 index 0000000..5bc39f6 --- /dev/null +++ b/.templates/setup.cfg.template @@ -0,0 +1,7 @@ +{% extends "templates/setup.cfg.template" %} + +{% block pytest %} +{{ super() }} +pytester_example_dir = {{ pytest.pytester_example_dir }} +python_files = {{ pytest.python_files }} +{% endblock %} diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f794694 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,97 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +language: python +python: 3.6 +notifications: + email: + on_success: change + on_failure: change +cache: pip + +dist: trusty + +env: + global: + - SCRIPT="test" + - TEST_ARGS="" + - COV_CORE_SOURCE=pytest_rng # early start pytest-cov engine + - COV_CORE_CONFIG=.coveragerc + - COV_CORE_DATAFILE=.coverage.eager + - BRANCH_NAME="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" + +jobs: + include: + - + env: + SCRIPT="static" + - + env: + SCRIPT="test-coverage" + - + env: + SCRIPT="test" + python: 3.5 + cache: False + - + env: + SCRIPT="test" + python: 3.7 + dist: xenial + - + env: + SCRIPT="docs" + addons: + apt: + packages: + - pandoc + +before_install: + # export travis_terminate for use in scripts + - export -f travis_terminate + _travis_terminate_linux + _travis_terminate_osx + _travis_terminate_unix + _travis_terminate_windows + # upgrade pip + - pip install pip --upgrade + # install/run nengo-bones + - pip install nengo-bones + - bones-generate --output-dir .ci ci-scripts + - if [[ "$TRAVIS_PYTHON_VERSION" < "3.6" ]]; then + echo "Skipping bones-check because Python $TRAVIS_PYTHON_VERSION < 3.6"; + else + bones-check; + fi + # display environment info + - pip freeze + +install: + - .ci/$SCRIPT.sh install + - pip freeze + +after_install: + - .ci/$SCRIPT.sh after_install + +before_script: + - .ci/$SCRIPT.sh before_script + +script: + - .ci/$SCRIPT.sh script + +before_cache: + - .ci/$SCRIPT.sh before_cache + +after_success: + - .ci/$SCRIPT.sh after_success + +after_failure: + - .ci/$SCRIPT.sh after_failure + +before_deploy: + - .ci/$SCRIPT.sh before_deploy + +after_deploy: + - .ci/$SCRIPT.sh after_deploy + +after_script: + - .ci/$SCRIPT.sh after_script diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..a005307 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,26 @@ +*************** +Release History +*************** + +.. Changelog entries should follow this format: + + version (release date) + ====================== + + **section** + + - One-line description of change (link to Github issue/PR) + +.. Changes should be organized in one of several sections: + + - Added + - Changed + - Deprecated + - Removed + - Fixed + +1.0.0 (unreleased) +================== + +Initial release of ``pytest-rng``! +Thanks to all of the contributors for making this possible! diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..08e5fab --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,46 @@ +.. Automatically generated by nengo-bones, do not edit this file directly + +************************** +Contributing to pytest-rng +************************** + +Issues and pull requests are always welcome! +We appreciate help from the community to make pytest-rng better. + +Filing issues +============= + +If you find a bug in pytest-rng, +or think that a certain feature is missing, +please consider +`filing an issue `_! +Please search the currently open issues first +to see if your bug or feature request already exists. +If so, feel free to add a comment to the issue +so that we know that multiple people are affected. + +Making pull requests +==================== + +If you want to fix a bug or add a feature to pytest-rng, +we welcome pull requests. +Ensure that you fill out all sections of the pull request template, +deleting the comments as you go. +We check most aspects of code style automatically. +Please refer to our +`code style guide `_ +for things that we check manually. + +Contributor agreement +===================== + +We require that all contributions be covered under +our contributor assignment agreement. Please see +`the agreement `_ +for instructions on how to sign. + +More details +============ + +For more details on how to contribute to Nengo, +please see the `developer guide `_. diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst new file mode 100644 index 0000000..386e3e7 --- /dev/null +++ b/CONTRIBUTORS.rst @@ -0,0 +1,9 @@ +.. Automatically generated by nengo-bones, do not edit this file directly + +*********************** +pytest-rng contributors +*********************** + +See https://github.com/nengo/pytest-rng/graphs/contributors +for a list of the people who have committed to pytest-rng. +Thank you for your contributions! diff --git a/LICENSE.rst b/LICENSE.rst new file mode 100644 index 0000000..940766c --- /dev/null +++ b/LICENSE.rst @@ -0,0 +1,29 @@ +.. Automatically generated by nengo-bones, do not edit this file directly + +****************** +pytest-rng license +****************** + +MIT License + +Copyright (c) 2019-2019 Applied Brain Research + +Permission is hereby granted, free of charge, +to any person obtaining a copy of this software +and associated documentation files (the "Software"), +to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7988735 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,37 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +global-include *.py +global-include *.sh +global-include *.template +include *.rst + +# Include files for CI and recreating the source dist +include *.yml +include *.yaml +include *.toml +include MANIFEST.in +include .gitlint +include .pylintrc + +# Directories to include +graft docs + +# Subdirectories to exclude, if they exist +prune docs/_build +prune dist +prune .git +prune .github +prune .tox +prune .eggs +prune .ci + +# Exclude auto-generated files +recursive-exclude docs *.py + +# Patterns to exclude from any directory +global-exclude *.ipynb_checkpoints* +global-exclude *-checkpoint.ipynb + +# Exclude all bytecode +global-exclude *.pyc *.pyo *.pyd + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..5179284 --- /dev/null +++ b/README.rst @@ -0,0 +1,87 @@ +********** +pytest-rng +********** + +``pytest-rng`` provides fixtures for +ensuring "randomness" in your tests is reproducible +from one run to the next. +It also allows the seed for all tests to be changed if requested, +to help ensure that test successes are not dependent on +particular random number seeds. + +- Use the ``rng`` fixture to get a pre-seeded random number generator (RNG) + that exposes NumPy's `~numpy.random.mtrand.RandomState` interface. + +- Use the ``seed`` fixture to get an integer seed + that can be used to initialize your own RNG. + +The following example prints the same four random numbers +every time the test is run. + +.. code-block:: python + + import numpy as np + + def test_rectification(rng, seed): + print(rng.uniform(-1, 1, size=3)) + print(seed) + +To use these fixtures, install with + +.. code-block:: bash + + pip install pytest-rng + +Once installed, you can use these fixtures like any other fixture: +add ``rng`` or ``seed`` to the arguments of a test function or class. + +Seed generation +=============== + +For the ``seed`` fixture, we generate a seed by doing the following: + +1. Concatenate the test's ``nodeid`` and a ``salt`` value, if provided. +2. Hash that string to yield an integer seed. + +For the ``rng`` fixture, we also add the string ``"rng"`` to the ``salt`` +value before generating the seed as above. +The seed is used to instantiate a `~numpy.random.mtrand.RandomState`, +which is returned. + +.. note:: We add ``"rng"`` to the salt to ensure that random numbers + are different when using the ``rng`` fixture + and when manually instantiating a ``RandomState`` + with the ``seed`` fixture. + +salt +==== + +``salt`` is a string that is added to the test's ``nodeid`` +in order to change the seed for all tests. +It is advantageous to change seeds regularly to ensure that +your test suite is robust to different seeds. + +The salt value can be specified in a configuration file +like ``setup.cfg`` or ``pytest.ini``. + +.. code-block:: ini + + [tool:pytest] + + rng_salt = v0.3.0 + +The salt value can also be specified through the command line. + +.. code-block:: bash + + pytest --rng-salt "v0.4.0" + +The salt value passed through the command line takes precedence +over the value set in the configuration file +so that you can change seeds on-the-fly. + +~~~~~ + +See the full +`documentation `__ +for more details. diff --git a/docs/_static/favicon.ico b/docs/_static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..d034ac95e66c9d18b06f7a127e9047daebdaa98d GIT binary patch literal 15086 zcmd5?3v^V)8NMNEutgK~^jLYAWOuX4W(nj)NFb1a4M{fHO`wKHwSd^Fw1N~5+WKNG zf+j&ydF3G}dGJaJNqtfESVNJAk5+3DP&_u_kpNO$1vHS5>G$8=33q4j&107h-E;o^ zXXeiTf8YFf?%bI>BS`^LpcES`L5E3OA|xqElBBROS3X3NzCu|DG+lq-8cAA)3Q?#7 z6KOVt%H^NIsg(a=VPWYyovy@Nf&B`-UO!gp)7!MgHsfe)Qqskk`1tm?gwfxE)1OFy zF-^-SuDuf;9?pCb_^8wJ-`nz%+{zS#IYNuaaV)z_hyo#Sy=as!lyh+xM`8%6(|2&ben7^w0*W1?LP;D!pTVpHR&v}zBP5m2tYT}11 z;g_RCzGD8W@ZXqUngpKfAm1r@Q<~yl7~R6Amds;@;RcbfnD00KoAcHTtFo0oQe`VU zC~~HJm9>JcO8Sr$nOfO{b6;X(A4?Uvs^$I4zqVlA&DD9O_k-U~NT(WaD%)V$z)Io{ z$lRaE*u!$R<*}$SCXw$YmFMn?^WT`W_NFRp*}WJ;-T=3YTAX?L`iyOC`S9jT6!)cx zhuM^uXR~q7q>J1sf68C{^*a6j8e8eF!D$Diopb9!Us|2A`%Cbaxfhy`v%8irVwQR< zGmVQD`Cq_%=QDrtPce>D0!u$CWqDRWC z|(rW40R?oVdyW!9~C%G@(IS(w4>G2WjG4Gq1`-~2}!N1Uv&u1#hV3*ekd zFN->M3u3-OWoMrs*B@ypqJTt_Nn@9+30>N2|CLqZ!mkzYS= z-b%g5o!T)XZ=}cC5dEf5FN$hS%dE}3S&jR+C24B&v1YT`Jm@O(Z!3&$ zwXZZB4$7|0{t38$O0iZ-=0;10&ZzfTha&bCHSYZu|ESg-&qp0I5?=1BxkD@s)`Lpk zu5<_H#=FtS#pkw0i@U%1$Ll8iP-t$wljCRC=Tm?Gj~e&*S&1bo+)oV|GUOI+ucrMu z{*Z5unX^4-IC!_HaBrVjlJP6l^_a(_-%R25TDm{@=WfqU0q_5+aBstVDi0i9fw=V> z%n>$eaaNZ5n}1eAt_8f$t8hP;U2mO)7|#{CBj0lS?YG~gBLmVQ^2b;+-5btI8-Mf1SYrgn z8Ie26OO!mkP5YC->-}^{sj_oY>R<=@BsJGVI{ zu)agTiRlYJUKjI?A+Vo`dpBDtC_4fXalTd7aiwRu`m(qCr-|L)`n!ANOyql8ZO#E5#pN`g(rjHSq2F=Ijl3vH|XT zA(lP$6_4x09q-Et=nK6%KLfmqTW`Jf$KG;Q+5s~W_$#30GcsOi`aY0hmlm@o`RjEW zYf2-8G}b6+JMtaDtW$F8HkWR4X_(OU4wr6nX>x^fnx&G4a_Z}(5ILXh)Bz4DR<>`H z7RcJ})Hh0!gYx*wC2MJ5fK!LK^z1-qzA;GFfp(`3mV)K{jn{a_L~F9u(@-16>;Na{62B(oEJ;0F#?61qwRInYU9s z@#QxPVF`nzLjvR!5+M-bhkHFA_p6~dg1#_UFz+^-pFiu1s1fE)Q%nrasTyGav}k~G zkM};xJe~!p^MOo?eM@y{HGxz%!1#G*p&Cc7wxHz0>4STu9l~LFYMb{s;y=3(thch z)OT6(bk#ldD~ew|rTjKvK8iCehxy*^+Odsn-kd6Cj5GSB{nW2;w%{(;`GT&!dingu z=nvS_ssCbWPu_7ETj#c(@T&bQvsVTozE+%T*n)PR=WXSBgQb#{#2##WF6IcEw)#P4 z8t=CMuUr%K3E!wUo~p8zZUoL*jwR>|n?wl7a-4R%#p);&~gK3c)|Yv$dzfze)`Db?-V$rHeCx&ruwxwpnv{xF^6S#E9} z1>6H1dr`q_GWM81xSiQK6&V?MgD?2Q4d(MT*0OmuQ(hcO&mHZF6i2r#18zIVe$28H z=WX&kvbN^1h>>pRM1M;*zTlHB`Ji7>)NM#JbUMxmhHAP^sV!bvPmpluT$>Db$yrh1>YEMJ`K#TmAFW^ zF1+i{3xIo4{!UDNZd|l-%--gVnDgHk{1Lj8uJ!`Alj1O=WI+n{m0D$Nw0-nLDc=Hv zk2hzB4I4Jt5BUG4`=9+6TO4q|=GYnQGf!jR|9g&)chql`*p&9A{bpUl_t?jLv^;@b zn_Xv32CjU+WqiZ{XG1^W*wDKGhnbK2&lh~MC2zEUkyCFy%iE1{=2&EMsPS_u{pS=>~jygxB7C9d?Db({C-xzPaysyz?kDLR6ZhYf_7xxYMGEW=Jm(X8!c<=O1s`!gt-xl;&sX~HO}Zz1 z`0Q@|p?s&V>m{C5xn=>rau12lp(@up^gQ)u{kO9Z{jE_MvmVlf-4p(y-tJLTeBiH7 z_;gQUqq_dawRa$4y@@-ogZR$T@+X#~?m~NLC%$ZubOhT&+kzde31Sa+bl`hO6GVax z2ncozg*XCH2C)YPJFbBY3=Ve0hN2A8m=^4KW--bjAuQMt2e}8b3-T-^7!t||BS1=n z*ToLW&e{U#J4Fa23=$y!BcbxeAKc%9@cYH}Td3{vo0dO!w;@uV3-99>VY>~o4?=yK zzDH2nZLUUjH`>Z4jI(5QA~rPMv~hc`sSM)w9<42?y{^huUVt^i&J~%jvdDxedN&Y` zrag`6Ro3!UtX)^Z?zFsq9QDyRx$|f9agOFys`e2&!w;}K4m(;OIbO=Bty^wBa`=Jy zrJZz^5%29*;@G`?>4uuzipfRhW52`P^81J8Y;vS79xr2qJMCL*+S9ed;Vq_OQ_F6e z8!wo(c~4etPKP1JK%Dm?RsqnO_J;7tHrQRj+^*%`N1lHU^NWjdQ{$Uqa|!SBc#3s* z_7UMxZ8(R$YI5bwS+H#%zih%k$QDwpxKEn+#CVcQNLiQJYnVL(*xpFUE55p(z#!t`OYQHRF$N0hcb*^_UI6ZCn= ztEjIw@72VI4XqiPA%}rYkDtd6*WB!p7oQ)x9w{0k4x3R<&lWvn^bG6+Mjf>B`8}fb z9!1^PQI|j8T)!Q7RFX(OULr{!1xivQWDibR?1MBWNYZ + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..29d7be1 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# +# Automatically generated by nengo-bones, do not edit this file directly + +import os + +import pytest_rng + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.githubpages", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "nbsphinx", + "nengo_sphinx_theme", + "numpydoc", +] + +# -- sphinx.ext.autodoc +autoclass_content = "both" # class and __init__ docstrings are concatenated +autodoc_default_options = {"members": None} +autodoc_member_order = "bysource" # default is alphabetical + +# -- sphinx.ext.intersphinx +intersphinx_mapping = { + "nengo": ("https://www.nengo.ai/nengo/", None), + "numpy": ("https://docs.scipy.org/doc/numpy", None), + "python": ("https://docs.python.org/3", None), +} + +# -- sphinx.ext.todo +todo_include_todos = True + +# -- numpydoc config +numpydoc_show_class_members = False + +# -- nbsphinx +nbsphinx_timeout = -1 + +# -- sphinx +nitpicky = True +exclude_patterns = ["_build", "**/.ipynb_checkpoints"] +linkcheck_timeout = 30 +source_suffix = ".rst" +source_encoding = "utf-8" +master_doc = "index" +linkcheck_ignore = [r"http://localhost:\d+"] +linkcheck_anchors = True +default_role = "py:obj" +pygments_style = "sphinx" + +project = "pytest-rng" +authors = "Applied Brain Research" +copyright = "2019-2019 Applied Brain Research" +version = ".".join(pytest_rng.__version__.split(".")[:2]) # Short X.Y version +release = pytest_rng.__version__ # Full version, with tags + +# -- HTML output +templates_path = ["_templates"] +html_static_path = ["_static"] +html_theme = "nengo_sphinx_theme" +html_title = "pytest-rng {0} docs".format(release) +htmlhelp_basename = "pytest-rng" +html_last_updated_fmt = "" # Default output format (suppressed) +html_show_sphinx = False +html_favicon = os.path.join("_static", "favicon.ico") +html_theme_options = { + "nengo_logo": "general-small-light.svg", + "nengo_logo_color": "#a8acaf", +} diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..b485363 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,9 @@ +.. include:: ../README.rst + :end-before: ~~~~ + +API reference +============= + +.. autofunction:: pytest_rng.plugin.rng + +.. autofunction:: pytest_rng.plugin.seed diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2410bfa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +[build-system] +requires = ["setuptools", "wheel"] + +[tool.black] +target-version = ['py35', 'py36', 'py37'] diff --git a/pytest_rng/__init__.py b/pytest_rng/__init__.py new file mode 100644 index 0000000..293b410 --- /dev/null +++ b/pytest_rng/__init__.py @@ -0,0 +1,6 @@ +"""pytest fixtures for seeding tests and making randomness reproducible.""" + +from .version import version as __version__ + +__copyright__ = "2019-2019 pytest_rng contributors" +__license__ = "MIT license" diff --git a/pytest_rng/plugin.py b/pytest_rng/plugin.py new file mode 100644 index 0000000..200d465 --- /dev/null +++ b/pytest_rng/plugin.py @@ -0,0 +1,59 @@ +import hashlib + +import numpy as np +import pytest + + +class Seed: + + salt = "" + + @classmethod + def generate(cls, uniqueid, extra_salt=""): + """Generate a unique seed for the given identifier. + + The seed should be the same across all machines/platforms. + """ + tohash = uniqueid + cls.salt + extra_salt + sha1 = hashlib.sha1(tohash.encode("utf-8")) + return int(sha1.hexdigest()[:8], 16) + + +def pytest_addoption(parser): + help_msg = "Specify string to salt `rng` and `seed` fixtures" + parser.addoption("--rng-salt", nargs=1, type=str, help=help_msg) + parser.addini("rng_salt", help=help_msg) + + +def pytest_configure(config): + cli_salt = config.getoption("--rng-salt", None) + ini_salt = config.getini("rng_salt") + if cli_salt is not None: + Seed.salt = cli_salt[0] + elif ini_salt is not None: + Seed.salt = ini_salt + + +@pytest.fixture +def rng(request): + """A seeded random number generator (RNG). + + An instance of `~numpy.random.mtrand.RandomState`. It is preferable + to the unseeded `numpy.random` because it is consistent from one + test run to the next when using the same salt value, and preferable to + a fixed seed RNG because changing the salt checks that tests + are not dependent on a specific seed. + """ + return np.random.RandomState(Seed.generate(request.node.nodeid, extra_salt="rng")) + + +@pytest.fixture +def seed(request): + """An integer random number generator seed in the range [0, 2**32 - 1]. + + The seed is consistent from one test run to the next + when using the same salt value. + It is preferable to a completely fixed seed because changing the salt + checks that tests are not dependent on a specific seed. + """ + return Seed.generate(request.node.nodeid) diff --git a/pytest_rng/tests/__init__.py b/pytest_rng/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pytest_rng/tests/test_fixtures.py b/pytest_rng/tests/test_fixtures.py new file mode 100644 index 0000000..bc32a4d --- /dev/null +++ b/pytest_rng/tests/test_fixtures.py @@ -0,0 +1,70 @@ +"""Test the `~.rng` and `~.seed` fixtures. + +To test that the values are consistent across multiple test runs, we require the +tests to be run twice. On the first test run, we cache randomly generated values. +On the second test run, we check that the previously cached values match the +values generated on this test run. +""" + +import numpy as np +import pytest + +from pytest_rng.plugin import Seed + + +@pytest.mark.parametrize("param", ["a", "b"]) +def test_seed(cache, param, seed): + fmt = "test_seed[{param}]:{salt}" + key = fmt.format(param=param, salt=Seed.salt) + + cached = cache.get(key, None) + if cached is not None: + # Seed should be the same across test runs + assert seed == cached + + # Different parametrizations should result in different seeds + other_key = fmt.format(param="a" if param == "b" else "b", salt=Seed.salt) + assert seed != cache.get(other_key, seed) + + else: + cache.set(key, seed) + assert cached, "Value was not yet cached, run again to test newly cached value" + + +@pytest.mark.parametrize("param", ["a", "b"]) +def test_seed_across_tests(cache, param, seed): + cached = cache.get( + "test_seed[{param}]:{salt}".format(param=param, salt=Seed.salt), None + ) + if cached is not None: + # This seed should not be the same as test_seed + assert seed != cached + + else: + assert cached, "Value was not yet cached, run again to test newly cached value" + + +@pytest.mark.parametrize("param", ["a", "b"]) +def test_rng(cache, param, rng): + fmt = "test_rng[{param}]:{salt}" + key = fmt.format(param=param, salt=Seed.salt) + vals = rng.rand(3).tolist() # Cache requires normal Python objects + + cached = cache.get(key, None) + if cached is not None: + + # Generated values should be the same across test runs + assert all(x == y for x, y in zip(vals, cached)) + + # Different parametrizations should result in different seeds + other_key = fmt.format(param="a" if param == "b" else "b", salt=Seed.salt) + assert not all(x == y for x, y in zip(vals, cache.get(other_key, vals))) + + else: + cache.set(key, vals) + assert cached, "Value was not yet cached, run again to test newly cached value" + + +def test_seed_rng(rng, seed): + manual_rng = np.random.RandomState(seed) + assert not np.all(rng.rand(3) == manual_rng.rand(3)) diff --git a/pytest_rng/tests/test_pytest.py b/pytest_rng/tests/test_pytest.py new file mode 100644 index 0000000..d242af8 --- /dev/null +++ b/pytest_rng/tests/test_pytest.py @@ -0,0 +1,110 @@ +"""The main test file that should be run on a regular basis. + +The goal of this file is to test pytest_rng internals and run all other +test files with various invocations of pytest to ensure that all possible +ways of using this plugin are tested. + +By default, if you call ``pytest`` while in this repository, only this +file will be run due to configuration in ``setup.cfg``. However, other +test files can be run manually by passing them to ``pytest``. +""" + +import os +from textwrap import dedent + +pytest_plugins = ["pytester"] + + +def assert_all_passed(result): + """Assert that all outcomes are 0 except for 'passed'. + + Also returns the number of passed tests. + """ + outcomes = result.parseoutcomes() + for outcome in outcomes: + if outcome not in ("passed", "seconds"): + assert outcomes[outcome] == 0 + return outcomes.get("passed", 0) + + +def copy_all_tests(testdir, path): + parts = path.strip("/").split("/") + for i in range(1, len(parts) + 1): + testdir.mkpydir("/".join(parts[:i])) + + # Find all test files in the current folder, not including this one. + # NB: If we add additional directories, this needs to change + tests = [ + p + for p in os.listdir(os.path.dirname(__file__)) + if p.startswith("test_") and p != "test_pytest.py" + ] + for test in tests: + test_path = testdir.copy_example(test) + test_path.rename("%s/%s" % (path, test)) + + +def check_consistency(testdir, *pytest_args): + """Run all tests twice to check seed consistency across runs.""" + + # The first pass fills up the cache + result = testdir.runpytest_subprocess("--cache-clear", *pytest_args) + + # The second pass should all pass + result = testdir.runpytest_subprocess(*pytest_args) + assert assert_all_passed(result) > 0 + + +def get_seeds_from_cache(testdir, salt): + """Run ``pytest --cache-show`` and parse the outcome to get cached seeds.""" + + result = testdir.runpytest("--cache-show") + fmt = "test_seed[{param}]:{salt} contains:" + + ix_a = result.outlines.index(fmt.format(param="a", salt=salt)) + seed_a = int(result.outlines[ix_a + 1].strip()) + + ix_b = result.outlines.index(fmt.format(param="b", salt=salt)) + seed_b = int(result.outlines[ix_b + 1].strip()) + + return seed_a, seed_b + + +def test_fixture_consistency(testdir): + copy_all_tests(testdir, "packages/tests") + check_consistency(testdir) + + +def test_salt(testdir): + copy_all_tests(testdir, "packages/tests") + + # First, test passing in a salt value via command line + check_consistency(testdir, "--rng-salt", "cli-salt") + + # Look at the cache to make sure the salt is being used + cli_seed_a, cli_seed_b = get_seeds_from_cache(testdir, salt="cli-salt") + assert cli_seed_a != cli_seed_b + + # Second, set up an ini config value + testdir.makeini( + dedent( + """\ + [pytest] + rng_salt = ini-salt + """ + ) + ) + # Check consistency without passing a salt + check_consistency(testdir) + + # Look at the cache to make sure the salt is being used + ini_seed_a, ini_seed_b = get_seeds_from_cache(testdir, salt="ini-salt") + assert ini_seed_a != ini_seed_b + assert ini_seed_a != cli_seed_a + + # Third, try passing the CLI salt again to test overriding + check_consistency(testdir, "--rng-salt", "cli-salt") + override_seed_a, override_seed_b = get_seeds_from_cache(testdir, salt="cli-salt") + assert override_seed_a != override_seed_b + assert override_seed_a != ini_seed_a + assert override_seed_a == cli_seed_a diff --git a/pytest_rng/version.py b/pytest_rng/version.py new file mode 100644 index 0000000..91006e8 --- /dev/null +++ b/pytest_rng/version.py @@ -0,0 +1,16 @@ +"""pytest-rng version information. + +We use semantic versioning (see http://semver.org/). +and conform to PEP440 (see https://www.python.org/dev/peps/pep-0440/). +'.devN' will be added to the version unless the code base represents +a release version. Release versions are git tagged with the version. +""" + +name = "pytest_rng" +version_info = (1, 0, 0) # (major, minor, patch) +dev = 0 + +version = "{v}{dev}".format( + v=".".join(str(v) for v in version_info), + dev=(".dev%d" % dev) if dev is not None else "", +) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..fd929cd --- /dev/null +++ b/setup.cfg @@ -0,0 +1,109 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +[build_sphinx] +source-dir = docs +build-dir = docs/_build +all_files = 1 + +[coverage:run] +source = pytest_rng + +[coverage:report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + # place ``# pragma: no cover`` at the end of a line to ignore it + pragma: no cover + + # Don't complain if tests don't hit defensive assertion code: + raise NotImplementedError + + # `pass` is just a placeholder, fine if it's not covered + ^[ \t]*pass$ + + +# Patterns for files to exclude from reporting +omit = + */tests/test* + +[flake8] +exclude = + __init__.py +ignore = + E123 + E133 + E203 + E226 + E241 + E242 + E501 + E731 + F401 + W503 +max-complexity = 10 +max-line-length = 88 + +[tool:pytest] +xfail_strict = False +norecursedirs = + .* + *.egg + build + dist + docs +filterwarnings = + ignore:testdir.copy_example is an experimental api +rng_salt = v1.0.0 +pytester_example_dir = pytest_rng/tests +python_files = test_pytest.py + +[pylint] +# note: pylint doesn't look in setup.cfg by default, need to call it with +# `pylint ... --rcfile=setup.cfg` +disable = + arguments-differ, + assignment-from-no-return, + attribute-defined-outside-init, + bad-continuation, + blacklisted-name, + comparison-with-callable, + duplicate-code, + fixme, + import-error, + invalid-name, + invalid-sequence-index, + len-as-condition, + literal-comparison, + no-else-raise, + no-else-return, + no-member, + no-name-in-module, + no-self-use, + not-an-iterable, + not-context-manager, + protected-access, + redefined-builtin, + stop-iteration-return, + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-return-statements, + too-many-statements, + unexpected-keyword-arg, + unidiomatic-typecheck, + unsubscriptable-object, + unsupported-assignment-operation, + unused-argument, + missing-docstring, +known-third-party = + matplotlib, + nengo, + numpy, + pytest, +max-line-length = 88 +valid-metaclass-classmethod-first-arg = metacls +reports = no +score = no diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3178df1 --- /dev/null +++ b/setup.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# Automatically generated by nengo-bones, do not edit this file directly + +import io +import os +import runpy + +try: + from setuptools import find_packages, setup +except ImportError: + raise ImportError( + "'setuptools' is required but not installed. To install it, " + "follow the instructions at " + "https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py" + ) + + +def read(*filenames, **kwargs): + encoding = kwargs.get("encoding", "utf-8") + sep = kwargs.get("sep", "\n") + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +root = os.path.dirname(os.path.realpath(__file__)) +version = runpy.run_path(os.path.join(root, "pytest_rng", "version.py"))["version"] + +install_req = ["numpy", "pytest"] +docs_req = ["nengo_sphinx_theme>=1.0", "sphinx"] +optional_req = [] +tests_req = [] + +setup( + name="pytest-rng", + version=version, + author="Applied Brain Research", + author_email="info@appliedbrainresearch.com", + packages=find_packages(), + url="https://www.nengo.ai/pytest-rng", + include_package_data=False, + license="MIT license", + description="Fixtures for seeding tests and making randomness reproducible", + long_description=read("README.rst", "CHANGES.rst"), + zip_safe=False, + install_requires=install_req, + extras_require={ + "all": docs_req + optional_req + tests_req, + "docs": docs_req, + "optional": optional_req, + "tests": tests_req, + }, + python_requires=">=3.5", + entry_points={"pytest11": ["rng = pytest_rng.plugin"]}, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Framework :: Pytest", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], +)