TRI DGP Open Source release

* Dataset-Governance-Policy (DGP) that codifies the
  schema and and maintenance of Autonomous Vehicle (AV)
  datasets developed by Toyota Research Institute (TRI).

.githooks/link_githooks.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Copyright 2019 Toyota Research Institute.  All rights reserved.
+chmod a+x .githooks/pre-commit
+ln -s -f ../../.githooks/pre-commit .git/hooks/pre-commit
+chmod a+x .githooks/pre-push
+ln -s -f ../../.githooks/pre-push .git/hooks/pre-push

.githooks/pre-commit

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Copyright 2019 Toyota Research Institute.  All rights reserved.
+
+# Git pre-commit hook to check staged Python files for formatting issues with
+# yapf.
+#
+# When running, this first checks for unstaged changes to staged files, and if
+# there are any, it will exit with an error. Files with unstaged changes will be
+# printed.
+#
+# If all staged files have no unstaged changes, it will run yapf against them,
+# leaving the formatting changes unstaged. Changed files will be printed.
+#
+# BUGS: This does not leave staged changes alone when used with the -a flag to
+# git commit, due to the fact that git stages ALL unstaged files when that flag
+# is used.
+
+
+# Find all staged Python files, and exit early if there aren't any.
+PYTHON_FILES=(`git diff --name-only --cached --diff-filter=AM | \
+  grep --color=never '.py$' | grep -Fv '_pb2'`)
+if [ ! "$PYTHON_FILES" ]; then
+  exit 0
+fi
+
+# Check for unstaged changes to files in the index.
+CHANGED_FILES=(`git diff --name-only ${PYTHON_FILES[@]}`)
+if [ "$CHANGED_FILES" ]; then
+  echo 'You have unstaged changes to some files in your commit; skipping '
+  echo 'auto-format. Please stage, stash, or revert these changes. You may '
+  echo 'find `git stash -k` helpful here.'
+  echo
+  echo 'Files with unstaged changes:'
+  for file in ${CHANGED_FILES[@]}; do
+    echo "  $file"
+  done
+  exit 1
+fi
+
+# Format all staged files, then exit with an error code if any have uncommitted
+# changes.
+echo 'Formatting staged Python files . . .'
+yapf --style .style.yapf -i -r ${PYTHON_FILES[@]}
+
+echo 'Fixing import order in Python files . . .'
+isort ${PYTHON_FILES[@]}
+
+
+CHANGED_FILES=(`git diff --name-only ${PYTHON_FILES[@]}`)
+if [ "$CHANGED_FILES" ]; then
+  echo 'Reformatted staged files. Please review and stage the changes.'
+  echo
+  echo 'Files updated:'
+  for file in ${CHANGED_FILES[@]}; do
+    echo "  $file"
+  done
+  exit 1
+else
+  exit 0
+fi

.githooks/pre-push

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright 2019 Toyota Research Institute.  All rights reserved.
+
+# Git pre-push hook to check commited Python files for pylint issues with
+# driving pylintrc. If there are any pylint errors, it will exit with an error.
+
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+# check if remote branch exists
+REMOTE=origin
+REMOTE_BRANCH=$REMOTE/$BRANCH
+git branch -a | grep $REMOTE_BRANCH
+if [ $? = 1 ]; then
+  REMOTE_BRANCH=$REMOTE/master
+fi
+
+COMMITED_PYTHON_FILES=$(git diff --name-only --diff-filter=ACM --name-only \
+                        $REMOTE_BRANCH..$BRANCH | grep --color=never '.py$')
+
+if [[ "$COMMITED_PYTHON_FILES" = "" ]]; then
+  echo 'No python files to lint.'
+else
+  pylint $COMMITED_PYTHON_FILES
+  if [ $? -ne 0 ]
+  then
+    echo "Aborting push due to files with lint."
+    exit 1
+  fi
+fi
+exit 0

.gitignore

@@ -0,0 +1,109 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest/*
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.idea/
+
+Pipfile
+Pipfile.lock
+
+\.vscode/

.pylintrc

@@ -0,0 +1,58 @@
+# Copyright 2019-2020 Toyota Research Institute.  All rights reserved.
+[MASTER]
+accept-no-param-doc=no
+accept-no-return-doc=yes
+accept-no-yields-doc=yes
+
+[REPORTS]
+reports=no
+# Make errors emacs-compatible
+msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'
+
+[TYPECHECK]
+ignored-classes=
+    PurePath
+
+[MESSAGES CONTROL]
+disable=
+   # We do not want lazy logging, as error during lazy logging are caught and
+   # ignored. Not a good recipe for reliable logging.
+   logging-not-lazy,
+   # also allow .format in logging calls
+   logging-format-interpolation,
+   # Do not enforce "refactor" rules
+   R,
+   C, duplicate-code,
+   # Temporary disable complexity checks.
+   too-many-instance-attributes, too-many-branches, too-many-statements,
+   too-many-arguments, too-many-locals,
+   # Do not complain on fixme/TODO's
+   fixme,
+   # Do not complain if we locally disabled a rule
+   locally-disabled,
+   # We do not care if we have too few public methods.
+   too-few-public-methods,
+   # We do not care if overridden methods use different arguments.
+   arguments-differ,
+   # Since this doesn't check control-flow, it has lots of false positives.
+   invalid-unary-operand-type, 
+   not-callable, 
+   no-member, 
+   protected-access, 
+   attribute-defined-outside-init,
+   global-statement,
+   W0123,
+   W1401,
+   # allow multiple arguments for string formatting
+   E1305,
+   # bypass import error in setup.py
+   E0401,
+   E0611,
+   # suppress Lambda warnings
+   W0108,
+   # allow un-implemented abstract method in sub-classes
+   W0223,
+   # allow explicit return in __init__
+   E0101,
+   # allow catching general exceptions
+   W0703

.style.yapf

@@ -0,0 +1,13 @@
+[style]
+based_on_style = pep8
+indent_width = 4
+column_limit = 120
+arithmetic_precedence_indication = false
+spaces_before_comment = 2
+split_complex_comprehension = true
+split_penalty_comprehension = 2100
+blank_line_before_nested_class_or_def = false
+align_closing_bracket_with_visual_indent = true
+dedent_closing_brackets = true
+coalesce_brackets = true
+join_multiple_lines = false

.travis.yml

@@ -0,0 +1,17 @@
+language: python
+jobs:
+  include:
+    - name: "Python 3.7 on Bionic Linux"
+      dist: bionic
+      python: 3.7
+    - name: "Python 3.6 on Bionic Linux"
+      dist: bionic
+      python: 3.6
+# command to install dependencies
+install:
+  - pip install cython==0.29.10
+  - pip install -r dev_requirements.txt
+# command to run tests
+script:
+  - cd $TRAVIS_BUILD_DIR
+  - make test

Dockerfile

@@ -0,0 +1,45 @@
+FROM pytorch/pytorch:1.4-cuda10.1-cudnn7-devel
+
+ARG python=3.6
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+ENV PYTHON_VERSION=${python}
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
+  build-essential \
+  ca-certificates \
+  libgl1-mesa-glx \
+  libgtk2.0-dev \
+  libjpeg-dev \
+  libpng-dev \
+  python${PYTHON_VERSION} \
+  python${PYTHON_VERSION}-dev
+RUN ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python
+RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
+  python get-pip.py && \
+  rm get-pip.py
+
+# Repo specific dependencies
+# Setup requirements in workspace
+ARG WORKSPACE=/home/dgp
+WORKDIR ${WORKSPACE}
+COPY dev_requirements.txt /tmp/
+RUN pip install cython==0.29.10 numpy==1.16.3
+RUN pip install -r /tmp/dev_requirements.txt --ignore-installed
+
+# Settings for S3
+RUN aws configure set default.s3.max_concurrent_requests 100 && \
+    aws configure set default.s3.max_queue_size 10000
+
+# Copy workspace and setup PYTHONPATH
+COPY . ${WORKSPACE}
+ENV PYTHONPATH="${WORKSPACE}:$PYTHONPATH"
+
+# Set up streamlit configs
+ARG STREAMLIT_CONFIG_DIR=/root/.streamlit
+RUN mkdir -p ${STREAMLIT_CONFIG_DIR} && \
+    touch ${STREAMLIT_CONFIG_DIR}/credentials.toml && \
+    printf '[general]\nemail = "dummy@domain.com"' > ${STREAMLIT_CONFIG_DIR}/credentials.toml

LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2019-2020 Toyota Research Institute. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include docs/*
+recursive-exclude * *.proto