Initial commit

.gitignore

@@ -0,0 +1,3 @@
+/.idea
+/lmu.egg-info
+*.pyc

.nengobones.yml

@@ -0,0 +1,56 @@
+project_name: LMU
+pkg_name: lmu
+repo_name: nengo/lmu
+description: LMU metapackage for installing various LMU implementations
+
+copyright_start: 2020
+
+license_rst: {}
+
+manifest_in: {}
+
+setup_py:
+  python_requires: ">=3.6"
+  url: https://github.com/nengo/lmu
+  install_req:
+    # minimum version specified because we import it in __init__
+    - keras-lmu>=0.3.0
+  tests_req:
+    - pytest>=6.1.0
+  classifiers:
+    - "Development Status :: 3 - Alpha"
+    - "Intended Audience :: Science/Research"
+    - "License :: Free for non-commercial use"
+    - "Operating System :: OS Independent"
+    - "Programming Language :: Python "
+    - "Programming Language :: Python :: 3.6"
+    - "Programming Language :: Python :: 3.7"
+    - "Programming Language :: Python :: 3.8"
+    - "Topic :: Scientific/Engineering "
+    - "Topic :: Scientific/Engineering :: Artificial Intelligence"
+
+travis_yml:
+  python: 3.8
+  jobs:
+    - script: test
+  pypi_user: __token__
+  slack_notifications: "A2NBzM/LLeoEiN1OHQdbjm5BKnBGHPx5lQkzW1zBg/i14Khjc7Y44U/0HOWJ\
+    fiVw/04vlwnwc7/vsFjpOv68g0bE/BcYBchtQTV5RN+6gkgpRHc9/zFfITYwKC414VEzDCQhxq7AD2ls\
+    cLaMGMO5UJ5DjWC4AP0URXYGIXoHVk8OnGdeJEVwBG/PCuGzTPQSqtCEO3hNUzQQ1T1QMtdxPKem9D3x\
+    5AkC0kYa8gxwtiHGf9mif7hdgg52v9SLmq8o9FBANgtEPBFNeMIriAEeBQaeK/zBMkX7SJgIHRiKgQmQ\
+    eL6cLxcm2eTIow3oBeMActU3jzxqfKqSjv3SHnHsQDaOeKvuqdC8/i4WNYJGbO0frxNJAOwE3QoYv/4P\
+    MkzGzQ6ymnNIEgGy2r1UHPqK2OzIp8sqr6P37Km0HvlGSHerAs/m4qe0mu+kVT1Amq4xg49DmephnPoc\
+    VhMHtWq0/iI9wiUYFc5bdISJdB8u8Uxk7xcJ5d6dAcW/ehSjPx53qE0P+brzuxHfe77t6GpP5r+8mvO/\
+    VkMc4Ewp9gv7LX4fHgSbEtQIxtlPiQjASPIl5PknC6GFmgcafuM85DEpQDVpqmueZK/HR4VvGF8BrzQi\
+    JzKf8mSOc9V3eYDaL5M0sK9wOK6SHXE7IkUcZGCopVJfd5chZUsWDKko5ZTZReg="
+  deploy_dists:
+    - sdist
+    - bdist_wheel
+
+ci_scripts:
+  - template: test
+  - template: deploy
+
+pre_commit_config_yaml: {}
+
+pyproject_toml: {}

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+# Automatically generated by nengo-bones, do not edit this file directly
+
+repos:
+-   repo: https://github.com/psf/black
+    rev: 20.8b0
+    hooks:
+    - id: black

.travis.yml

@@ -0,0 +1,104 @@
+# Automatically generated by nengo-bones, do not edit this file directly
+
+language: python
+python: 3.8
+notifications:
+  email:
+    on_success: change
+    on_failure: change
+  slack:
+    if: branch = master
+    on_pull_requests: false
+    on_success: change
+    on_failure: always
+    rooms:
+      - secure: "A2NBzM/LLeoEiN1OHQdbjm5BKnBGHPx5lQkzW1zBg/i14Khjc7Y44U/0HOWJfiVw/04vlwnwc7/vsFjpOv68g0bE/BcYBchtQTV5RN+6gkgpRHc9/zFfITYwKC414VEzDCQhxq7AD2lscLaMGMO5UJ5DjWC4AP0URXYGIXoHVk8OnGdeJEVwBG/PCuGzTPQSqtCEO3hNUzQQ1T1QMtdxPKem9D3x5AkC0kYa8gxwtiHGf9mif7hdgg52v9SLmq8o9FBANgtEPBFNeMIriAEeBQaeK/zBMkX7SJgIHRiKgQmQeL6cLxcm2eTIow3oBeMActU3jzxqfKqSjv3SHnHsQDaOeKvuqdC8/i4WNYJGbO0frxNJAOwE3QoYv/4PMkzGzQ6ymnNIEgGy2r1UHPqK2OzIp8sqr6P37Km0HvlGSHerAs/m4qe0mu+kVT1Amq4xg49DmephnPocVhMHtWq0/iI9wiUYFc5bdISJdB8u8Uxk7xcJ5d6dAcW/ehSjPx53qE0P+brzuxHfe77t6GpP5r+8mvO/VkMc4Ewp9gv7LX4fHgSbEtQIxtlPiQjASPIl5PknC6GFmgcafuM85DEpQDVpqmueZK/HR4VvGF8BrzQiJzKf8mSOc9V3eYDaL5M0sK9wOK6SHXE7IkUcZGCopVJfd5chZUsWDKko5ZTZReg="
+cache: pip
+
+dist: xenial
+
+env:
+  global:
+    - SCRIPT="test"
+    - TEST_ARGS=""
+    - BRANCH_NAME="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}"
+    - PIP_USE_FEATURE="2020-resolver"
+
+jobs:
+  include:
+  -
+    env:
+      SCRIPT="test"
+  - stage: deploy
+    if: branch =~ ^release-candidate-* OR tag =~ ^v[0-9]*
+    env: SCRIPT="deploy"
+    cache: false
+    deploy:
+      - provider: pypi
+        server: https://test.pypi.org/legacy/
+        user: __token__
+        password: $PYPI_TEST_TOKEN
+        distributions: "sdist bdist_wheel "
+        on:
+          all_branches: true
+          tags: false
+          condition: $TRAVIS_BRANCH =~ ^release-candidate-*
+      - provider: pypi
+        user: __token__
+        password: $PYPI_TOKEN
+        distributions: "sdist bdist_wheel "
+        on:
+          all_branches: true
+          tags: true
+          condition: $TRAVIS_TAG =~ ^v[0-9]*
+
+before_install:
+  # export travis_terminate for use in scripts, from here:
+  # https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_terminate.bash
+  - export -f travis_terminate
+    _travis_terminate_agent
+    _travis_terminate_freebsd
+    _travis_terminate_linux
+    _travis_terminate_osx
+    _travis_terminate_unix
+    _travis_terminate_windows
+  # upgrade pip
+  - pip install pip --upgrade
+  # install/run nengo-bones
+  - pip install git+https://github.com/nengo/nengo-bones#egg=nengo-bones
+  - bones-generate --output-dir .ci ci-scripts
+  - if [[ "$TRAVIS_PYTHON_VERSION" < "3.6" ]]; then
+        echo "Skipping bones-check because Python $TRAVIS_PYTHON_VERSION < 3.6";
+    else
+        bones-check --verbose;
+    fi
+  # display environment info
+  - pip freeze
+
+install:
+  - .ci/$SCRIPT.sh install
+  - pip freeze
+
+before_script:
+  - .ci/$SCRIPT.sh before_script
+
+script:
+  - .ci/$SCRIPT.sh script
+
+before_cache:
+  - .ci/$SCRIPT.sh before_cache
+
+after_success:
+  - .ci/$SCRIPT.sh after_success
+
+after_failure:
+  - .ci/$SCRIPT.sh after_failure
+
+before_deploy:
+  - .ci/$SCRIPT.sh before_deploy
+
+after_deploy:
+  - .ci/$SCRIPT.sh after_deploy
+
+after_script:
+  - .ci/$SCRIPT.sh after_script

CHANGES.rst

@@ -0,0 +1,27 @@
+***************
+Release history
+***************
+
+.. Changelog entries should follow this format:
+
+   version (release date)
+   ======================
+
+   **section**
+
+   - One-line description of change (link to Github issue/PR)
+
+.. Changes should be organized in one of several sections:
+
+   - Added
+   - Changed
+   - Deprecated
+   - Removed
+   - Fixed
+
+0.3.0 (unreleased)
+==================
+
+The version of this package as it existed prior to 0.3.0 has been renamed
+to ``keras-lmu``. This package will continue as a metapackage for installing
+multiple LMU implementations.

LICENSE.rst

@@ -0,0 +1,25 @@
+.. Automatically generated by nengo-bones, do not edit this file directly
+
+***********
+LMU license
+***********
+
+Copyright (c) 2020-2020 Applied Brain Research
+
+LMU is made available under a proprietary license
+that permits using, copying, sharing, and making derivative works from
+LMU and its source code for any non-commercial purpose,
+as long as the above copyright notice and this permission notice
+are included in all copies or substantial portions of the software.
+
+If you would like to use LMU commercially,
+licenses can be purchased from Applied Brain Research.
+Please contact info@appliedbrainresearch.com for more information.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -0,0 +1,37 @@
+# Automatically generated by nengo-bones, do not edit this file directly
+
+global-include *.py
+global-include *.sh
+global-include *.template
+include *.rst
+
+# Include files for CI and recreating the source dist
+include *.yml
+include *.yaml
+include *.toml
+include MANIFEST.in
+include .gitlint
+include .pylintrc
+
+# Directories to include
+graft docs
+
+# Subdirectories to exclude, if they exist
+prune docs/_build
+prune dist
+prune .git
+prune .github
+prune .tox
+prune .eggs
+prune .ci
+
+# Exclude auto-generated files
+recursive-exclude docs *.py
+
+# Patterns to exclude from any directory
+global-exclude *.ipynb_checkpoints*
+global-exclude *-checkpoint.ipynb
+
+# Exclude all bytecode
+global-exclude *.pyc *.pyo *.pyd
+

README.rst

@@ -0,0 +1,76 @@
+Legendre Memory Units: Continuous-Time Representation in Recurrent Neural Networks
+----------------------------------------------------------------------------------
+
+.. note::
+
+   This is a metapackage for installing LMU implementations. It does not contain any
+   code itself. See the list of implementations below.
+
+`Paper <https://papers.nips.cc/paper/9689-legendre-memory-units-continuous-time-representation-in-recurrent-neural-networks.pdf>`_
+
+The LMU is a novel memory cell for recurrent neural
+networks that dynamically maintains information across long windows of time using
+relatively few resources. It has been shown to perform as well as standard LSTM or
+other RNN-based models in a variety of tasks, generally with fewer internal parameters
+(see `this paper
+<https://papers.nips.cc/paper/9689-legendre-memory-units-continuous-time-representation-in-recurrent-neural-networks.pdf>`_ for more details). For the Permuted Sequential MNIST (psMNIST) task in particular, it has been demonstrated to outperform the current state-of-the-art results. See the note below for instructions on how to get access to this model.
+
+The LMU is mathematically derived to orthogonalize its continuous-time history – doing
+so by solving *d* coupled ordinary differential equations (ODEs), whose phase space
+linearly maps onto sliding windows of time via the Legendre polynomials up to degree
+*d* − 1 (the example for *d* = 12 is shown below).
+
+.. image:: https://i.imgur.com/Uvl6tj5.png
+   :target: https://i.imgur.com/Uvl6tj5.png
+   :alt: Legendre polynomials
+
+A single LMU cell expresses the following computational graph, which takes in an input
+signal, **x**, and couples a optimal linear memory, **m**, with a nonlinear hidden
+state, **h**. By default, this coupling is trained via backpropagation, while the
+dynamics of the memory remain fixed.
+
+.. image:: https://i.imgur.com/IJGUVg6.png
+   :target: https://i.imgur.com/IJGUVg6.png
+   :alt: Computational graph
+
+The discretized **A** and **B** matrices are initialized according to the LMU's
+mathematical derivation with respect to some chosen window length, **θ**.
+Backpropagation can be used to learn this time-scale, or fine-tune **A** and **B**,
+if necessary.
+
+Both the kernels, **W**, and the encoders, **e**, are learned. Intuitively, the kernels
+learn to compute nonlinear functions across the memory, while the encoders learn to
+project the relevant information into the memory (see `paper
+<https://papers.nips.cc/paper/9689-legendre-memory-units-continuous-time-representation-in-recurrent-neural-networks.pdf>`_ for details).
+
+LMU implementations
+-------------------
+
+* `KerasLMU <https://www.nengo.ai/keras-lmu>`_: Implementation of LMUs in Keras (this
+  is the original LMU implementation, which used to be referred to generically
+  as *the* LMU repo).
+
+Examples
+--------
+
+* `State of the art performance on psMNIST using KerasLMU
+  <https://www.nengo.ai/keras-lmu/examples/psMNIST.html>`_
+* `LMUs in Nengo (with online learning)
+  <https://www.nengo.ai/nengo/examples/learning/lmu.html>`_
+* `Spiking LMUs in Nengo Loihi (with online learning)
+  <https://www.nengo.ai/nengo-loihi/examples/lmu.html>`_
+* `LMUs in NengoDL (reproducing SotA on psMNIST)
+  <https://www.nengo.ai/nengo-dl/examples/lmu.html>`_
+
+Citation
+--------
+
+.. code-block::
+
+   @inproceedings{voelker2019lmu,
+     title={Legendre Memory Units: Continuous-Time Representation in Recurrent Neural Networks},
+     author={Aaron R. Voelker and Ivana Kaji\'c and Chris Eliasmith},
+     booktitle={Advances in Neural Information Processing Systems},
+     pages={15544--15553},
+     year={2019}
+   }

lmu/__init__.py

@@ -0,0 +1,13 @@
+"""LMU metapackage (should not be imported directly)."""
+
+import sys
+
+from keras_lmu.layers import LMU, LMUCell, LMUFFT
+
+print(
+    "'lmu' is a metapackage that should not be imported directly. You might be "
+    "intending to `import keras_lmu` (which used to be generically referred to as "
+    "'lmu'). For now this will have the effect of importing 'keras_lmu', but this will "
+    "not work in the future so you should update your imports.",
+    file=sys.stderr,
+)

lmu/tests/test_lmu.py

@@ -0,0 +1,16 @@
+from importlib import reload
+
+import keras_lmu
+import lmu
+
+
+def test_import_fallback(capsys):
+    reload(lmu)
+
+    assert (
+        "'lmu' is a metapackage that should not be imported" in capsys.readouterr().err
+    )
+
+    assert lmu.LMU is keras_lmu.LMU
+    assert lmu.LMUCell is keras_lmu.LMUCell
+    assert lmu.LMUFFT is keras_lmu.LMUFFT

lmu/version.py

@@ -0,0 +1,15 @@
+"""LMU version information.
+
+We use semantic versioning (see http://semver.org/).
+and conform to PEP440 (see https://www.python.org/dev/peps/pep-0440/).
+'.devN' will be added to the version unless the code base represents
+a release version. Release versions are git tagged with the version.
+"""
+
+name = "lmu"
+version_info = (0, 3, 0)  # (major, minor, patch)
+dev = 0
+
+version = (
+    f"{'.'.join(str(v) for v in version_info)}{f'.dev{dev}' if dev is not None else ''}"
+)

pyproject.toml

@@ -0,0 +1,7 @@
+# Automatically generated by nengo-bones, do not edit this file directly
+
+[build-system]
+requires = ["setuptools", "wheel"]
+
+[tool.black]
+target-version = ['py35', 'py36', 'py37', 'py38']