From 80ab2b5f85d58277e8e13b55ff24245ea2527b4b Mon Sep 17 00:00:00 2001 From: Daniel Rasmussen Date: Thu, 5 Nov 2020 17:10:59 -0400 Subject: [PATCH] Initial commit --- .gitignore | 3 ++ .nengobones.yml | 56 ++++++++++++++++++++++ .pre-commit-config.yaml | 7 +++ .travis.yml | 104 ++++++++++++++++++++++++++++++++++++++++ CHANGES.rst | 27 +++++++++++ LICENSE.rst | 25 ++++++++++ MANIFEST.in | 37 ++++++++++++++ README.rst | 76 +++++++++++++++++++++++++++++ lmu/__init__.py | 13 +++++ lmu/tests/__init__.py | 0 lmu/tests/test_lmu.py | 16 +++++++ lmu/version.py | 15 ++++++ pyproject.toml | 7 +++ setup.py | 73 ++++++++++++++++++++++++++++ 14 files changed, 459 insertions(+) create mode 100644 .gitignore create mode 100644 .nengobones.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .travis.yml create mode 100644 CHANGES.rst create mode 100644 LICENSE.rst create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 lmu/__init__.py create mode 100644 lmu/tests/__init__.py create mode 100644 lmu/tests/test_lmu.py create mode 100644 lmu/version.py create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8172d23 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/.idea +/lmu.egg-info +*.pyc diff --git a/.nengobones.yml b/.nengobones.yml new file mode 100644 index 0000000..5b12dab --- /dev/null +++ b/.nengobones.yml @@ -0,0 +1,56 @@ +project_name: LMU +pkg_name: lmu +repo_name: nengo/lmu +description: LMU metapackage for installing various LMU implementations + +copyright_start: 2020 + +license_rst: {} + +manifest_in: {} + +setup_py: + python_requires: ">=3.6" + url: https://github.com/nengo/lmu + install_req: + # minimum version specified because we import it in __init__ + - keras-lmu>=0.3.0 + tests_req: + - pytest>=6.1.0 + classifiers: + - "Development Status :: 3 - Alpha" + - "Intended Audience :: Science/Research" + - "License :: Free for non-commercial use" + - "Operating System :: OS Independent" + - "Programming Language :: Python " + - "Programming Language :: Python :: 3.6" + - "Programming Language :: Python :: 3.7" + - "Programming Language :: Python :: 3.8" + - "Topic :: Scientific/Engineering " + - "Topic :: Scientific/Engineering :: Artificial Intelligence" + +travis_yml: + python: 3.8 + jobs: + - script: test + pypi_user: __token__ + slack_notifications: "A2NBzM/LLeoEiN1OHQdbjm5BKnBGHPx5lQkzW1zBg/i14Khjc7Y44U/0HOWJ\ + fiVw/04vlwnwc7/vsFjpOv68g0bE/BcYBchtQTV5RN+6gkgpRHc9/zFfITYwKC414VEzDCQhxq7AD2ls\ + cLaMGMO5UJ5DjWC4AP0URXYGIXoHVk8OnGdeJEVwBG/PCuGzTPQSqtCEO3hNUzQQ1T1QMtdxPKem9D3x\ + 5AkC0kYa8gxwtiHGf9mif7hdgg52v9SLmq8o9FBANgtEPBFNeMIriAEeBQaeK/zBMkX7SJgIHRiKgQmQ\ + eL6cLxcm2eTIow3oBeMActU3jzxqfKqSjv3SHnHsQDaOeKvuqdC8/i4WNYJGbO0frxNJAOwE3QoYv/4P\ + MkzGzQ6ymnNIEgGy2r1UHPqK2OzIp8sqr6P37Km0HvlGSHerAs/m4qe0mu+kVT1Amq4xg49DmephnPoc\ + VhMHtWq0/iI9wiUYFc5bdISJdB8u8Uxk7xcJ5d6dAcW/ehSjPx53qE0P+brzuxHfe77t6GpP5r+8mvO/\ + VkMc4Ewp9gv7LX4fHgSbEtQIxtlPiQjASPIl5PknC6GFmgcafuM85DEpQDVpqmueZK/HR4VvGF8BrzQi\ + JzKf8mSOc9V3eYDaL5M0sK9wOK6SHXE7IkUcZGCopVJfd5chZUsWDKko5ZTZReg=" + deploy_dists: + - sdist + - bdist_wheel + +ci_scripts: + - template: test + - template: deploy + +pre_commit_config_yaml: {} + +pyproject_toml: {} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3117066 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +repos: +- repo: https://github.com/psf/black + rev: 20.8b0 + hooks: + - id: black diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..52d61a8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,104 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +language: python +python: 3.8 +notifications: + email: + on_success: change + on_failure: change + slack: + if: branch = master + on_pull_requests: false + on_success: change + on_failure: always + rooms: + - secure: "A2NBzM/LLeoEiN1OHQdbjm5BKnBGHPx5lQkzW1zBg/i14Khjc7Y44U/0HOWJfiVw/04vlwnwc7/vsFjpOv68g0bE/BcYBchtQTV5RN+6gkgpRHc9/zFfITYwKC414VEzDCQhxq7AD2lscLaMGMO5UJ5DjWC4AP0URXYGIXoHVk8OnGdeJEVwBG/PCuGzTPQSqtCEO3hNUzQQ1T1QMtdxPKem9D3x5AkC0kYa8gxwtiHGf9mif7hdgg52v9SLmq8o9FBANgtEPBFNeMIriAEeBQaeK/zBMkX7SJgIHRiKgQmQeL6cLxcm2eTIow3oBeMActU3jzxqfKqSjv3SHnHsQDaOeKvuqdC8/i4WNYJGbO0frxNJAOwE3QoYv/4PMkzGzQ6ymnNIEgGy2r1UHPqK2OzIp8sqr6P37Km0HvlGSHerAs/m4qe0mu+kVT1Amq4xg49DmephnPocVhMHtWq0/iI9wiUYFc5bdISJdB8u8Uxk7xcJ5d6dAcW/ehSjPx53qE0P+brzuxHfe77t6GpP5r+8mvO/VkMc4Ewp9gv7LX4fHgSbEtQIxtlPiQjASPIl5PknC6GFmgcafuM85DEpQDVpqmueZK/HR4VvGF8BrzQiJzKf8mSOc9V3eYDaL5M0sK9wOK6SHXE7IkUcZGCopVJfd5chZUsWDKko5ZTZReg=" +cache: pip + +dist: xenial + +env: + global: + - SCRIPT="test" + - TEST_ARGS="" + - BRANCH_NAME="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" + - PIP_USE_FEATURE="2020-resolver" + +jobs: + include: + - + env: + SCRIPT="test" + - stage: deploy + if: branch =~ ^release-candidate-* OR tag =~ ^v[0-9]* + env: SCRIPT="deploy" + cache: false + deploy: + - provider: pypi + server: https://test.pypi.org/legacy/ + user: __token__ + password: $PYPI_TEST_TOKEN + distributions: "sdist bdist_wheel " + on: + all_branches: true + tags: false + condition: $TRAVIS_BRANCH =~ ^release-candidate-* + - provider: pypi + user: __token__ + password: $PYPI_TOKEN + distributions: "sdist bdist_wheel " + on: + all_branches: true + tags: true + condition: $TRAVIS_TAG =~ ^v[0-9]* + +before_install: + # export travis_terminate for use in scripts, from here: + # https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_terminate.bash + - export -f travis_terminate + _travis_terminate_agent + _travis_terminate_freebsd + _travis_terminate_linux + _travis_terminate_osx + _travis_terminate_unix + _travis_terminate_windows + # upgrade pip + - pip install pip --upgrade + # install/run nengo-bones + - pip install git+https://github.com/nengo/nengo-bones#egg=nengo-bones + - bones-generate --output-dir .ci ci-scripts + - if [[ "$TRAVIS_PYTHON_VERSION" < "3.6" ]]; then + echo "Skipping bones-check because Python $TRAVIS_PYTHON_VERSION < 3.6"; + else + bones-check --verbose; + fi + # display environment info + - pip freeze + +install: + - .ci/$SCRIPT.sh install + - pip freeze + +before_script: + - .ci/$SCRIPT.sh before_script + +script: + - .ci/$SCRIPT.sh script + +before_cache: + - .ci/$SCRIPT.sh before_cache + +after_success: + - .ci/$SCRIPT.sh after_success + +after_failure: + - .ci/$SCRIPT.sh after_failure + +before_deploy: + - .ci/$SCRIPT.sh before_deploy + +after_deploy: + - .ci/$SCRIPT.sh after_deploy + +after_script: + - .ci/$SCRIPT.sh after_script diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..22ce6f7 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,27 @@ +*************** +Release history +*************** + +.. Changelog entries should follow this format: + + version (release date) + ====================== + + **section** + + - One-line description of change (link to Github issue/PR) + +.. Changes should be organized in one of several sections: + + - Added + - Changed + - Deprecated + - Removed + - Fixed + +0.3.0 (unreleased) +================== + +The version of this package as it existed prior to 0.3.0 has been renamed +to ``keras-lmu``. This package will continue as a metapackage for installing +multiple LMU implementations. diff --git a/LICENSE.rst b/LICENSE.rst new file mode 100644 index 0000000..f54b1aa --- /dev/null +++ b/LICENSE.rst @@ -0,0 +1,25 @@ +.. Automatically generated by nengo-bones, do not edit this file directly + +*********** +LMU license +*********** + +Copyright (c) 2020-2020 Applied Brain Research + +LMU is made available under a proprietary license +that permits using, copying, sharing, and making derivative works from +LMU and its source code for any non-commercial purpose, +as long as the above copyright notice and this permission notice +are included in all copies or substantial portions of the software. + +If you would like to use LMU commercially, +licenses can be purchased from Applied Brain Research. +Please contact info@appliedbrainresearch.com for more information. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7988735 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,37 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +global-include *.py +global-include *.sh +global-include *.template +include *.rst + +# Include files for CI and recreating the source dist +include *.yml +include *.yaml +include *.toml +include MANIFEST.in +include .gitlint +include .pylintrc + +# Directories to include +graft docs + +# Subdirectories to exclude, if they exist +prune docs/_build +prune dist +prune .git +prune .github +prune .tox +prune .eggs +prune .ci + +# Exclude auto-generated files +recursive-exclude docs *.py + +# Patterns to exclude from any directory +global-exclude *.ipynb_checkpoints* +global-exclude *-checkpoint.ipynb + +# Exclude all bytecode +global-exclude *.pyc *.pyo *.pyd + diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f99198d --- /dev/null +++ b/README.rst @@ -0,0 +1,76 @@ +Legendre Memory Units: Continuous-Time Representation in Recurrent Neural Networks +---------------------------------------------------------------------------------- + +.. note:: + + This is a metapackage for installing LMU implementations. It does not contain any + code itself. See the list of implementations below. + +`Paper `_ + +The LMU is a novel memory cell for recurrent neural +networks that dynamically maintains information across long windows of time using +relatively few resources. It has been shown to perform as well as standard LSTM or +other RNN-based models in a variety of tasks, generally with fewer internal parameters +(see `this paper +`_ for more details). For the Permuted Sequential MNIST (psMNIST) task in particular, it has been demonstrated to outperform the current state-of-the-art results. See the note below for instructions on how to get access to this model. + +The LMU is mathematically derived to orthogonalize its continuous-time history – doing +so by solving *d* coupled ordinary differential equations (ODEs), whose phase space +linearly maps onto sliding windows of time via the Legendre polynomials up to degree +*d* − 1 (the example for *d* = 12 is shown below). + +.. image:: https://i.imgur.com/Uvl6tj5.png + :target: https://i.imgur.com/Uvl6tj5.png + :alt: Legendre polynomials + +A single LMU cell expresses the following computational graph, which takes in an input +signal, **x**, and couples a optimal linear memory, **m**, with a nonlinear hidden +state, **h**. By default, this coupling is trained via backpropagation, while the +dynamics of the memory remain fixed. + +.. image:: https://i.imgur.com/IJGUVg6.png + :target: https://i.imgur.com/IJGUVg6.png + :alt: Computational graph + +The discretized **A** and **B** matrices are initialized according to the LMU's +mathematical derivation with respect to some chosen window length, **θ**. +Backpropagation can be used to learn this time-scale, or fine-tune **A** and **B**, +if necessary. + +Both the kernels, **W**, and the encoders, **e**, are learned. Intuitively, the kernels +learn to compute nonlinear functions across the memory, while the encoders learn to +project the relevant information into the memory (see `paper +`_ for details). + +LMU implementations +------------------- + +* `KerasLMU `_: Implementation of LMUs in Keras (this + is the original LMU implementation, which used to be referred to generically + as *the* LMU repo). + +Examples +-------- + +* `State of the art performance on psMNIST using KerasLMU + `_ +* `LMUs in Nengo (with online learning) + `_ +* `Spiking LMUs in Nengo Loihi (with online learning) + `_ +* `LMUs in NengoDL (reproducing SotA on psMNIST) + `_ + +Citation +-------- + +.. code-block:: + + @inproceedings{voelker2019lmu, + title={Legendre Memory Units: Continuous-Time Representation in Recurrent Neural Networks}, + author={Aaron R. Voelker and Ivana Kaji\'c and Chris Eliasmith}, + booktitle={Advances in Neural Information Processing Systems}, + pages={15544--15553}, + year={2019} + } diff --git a/lmu/__init__.py b/lmu/__init__.py new file mode 100644 index 0000000..572a998 --- /dev/null +++ b/lmu/__init__.py @@ -0,0 +1,13 @@ +"""LMU metapackage (should not be imported directly).""" + +import sys + +from keras_lmu.layers import LMU, LMUCell, LMUFFT + +print( + "'lmu' is a metapackage that should not be imported directly. You might be " + "intending to `import keras_lmu` (which used to be generically referred to as " + "'lmu'). For now this will have the effect of importing 'keras_lmu', but this will " + "not work in the future so you should update your imports.", + file=sys.stderr, +) diff --git a/lmu/tests/__init__.py b/lmu/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lmu/tests/test_lmu.py b/lmu/tests/test_lmu.py new file mode 100644 index 0000000..54cb354 --- /dev/null +++ b/lmu/tests/test_lmu.py @@ -0,0 +1,16 @@ +from importlib import reload + +import keras_lmu +import lmu + + +def test_import_fallback(capsys): + reload(lmu) + + assert ( + "'lmu' is a metapackage that should not be imported" in capsys.readouterr().err + ) + + assert lmu.LMU is keras_lmu.LMU + assert lmu.LMUCell is keras_lmu.LMUCell + assert lmu.LMUFFT is keras_lmu.LMUFFT diff --git a/lmu/version.py b/lmu/version.py new file mode 100644 index 0000000..9f37a3b --- /dev/null +++ b/lmu/version.py @@ -0,0 +1,15 @@ +"""LMU version information. + +We use semantic versioning (see http://semver.org/). +and conform to PEP440 (see https://www.python.org/dev/peps/pep-0440/). +'.devN' will be added to the version unless the code base represents +a release version. Release versions are git tagged with the version. +""" + +name = "lmu" +version_info = (0, 3, 0) # (major, minor, patch) +dev = 0 + +version = ( + f"{'.'.join(str(v) for v in version_info)}{f'.dev{dev}' if dev is not None else ''}" +) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c7f5d32 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +# Automatically generated by nengo-bones, do not edit this file directly + +[build-system] +requires = ["setuptools", "wheel"] + +[tool.black] +target-version = ['py35', 'py36', 'py37', 'py38'] diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f900ed1 --- /dev/null +++ b/setup.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# Automatically generated by nengo-bones, do not edit this file directly + +import io +import os +import runpy + +try: + from setuptools import find_packages, setup +except ImportError: + raise ImportError( + "'setuptools' is required but not installed. To install it, " + "follow the instructions at " + "https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py" + ) + + +def read(*filenames, **kwargs): + encoding = kwargs.get("encoding", "utf-8") + sep = kwargs.get("sep", "\n") + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +root = os.path.dirname(os.path.realpath(__file__)) +version = runpy.run_path(os.path.join(root, "lmu", "version.py"))["version"] + +install_req = [ + "keras-lmu>=0.3.0", +] +docs_req = [] +optional_req = [] +tests_req = [ + "pytest>=6.1.0", +] + +setup( + name="lmu", + version=version, + author="Applied Brain Research", + author_email="info@appliedbrainresearch.com", + packages=find_packages(), + url="https://github.com/nengo/lmu", + include_package_data=False, + license="Free for non-commercial use", + description="LMU metapackage for installing various LMU implementations", + long_description=read("README.rst", "CHANGES.rst"), + zip_safe=False, + install_requires=install_req, + extras_require={ + "all": docs_req + optional_req + tests_req, + "docs": docs_req, + "optional": optional_req, + "tests": tests_req, + }, + python_requires=">=3.6", + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "License :: Free for non-commercial use", + "Operating System :: OS Independent", + "Programming Language :: Python ", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Topic :: Scientific/Engineering ", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + ], +)