v1.0 release

This release includes v1.0 components from TRI internal DGP: - Protos - DatasetClass (Synchronized, Frame) - CLI with visualizer
TRI-ML · Sep 30, 2021 · 6ff13df · 6ff13df
1 parent ff7da7a
commit 6ff13df
Show file tree

Hide file tree

Showing 93 changed files with 9,785 additions and 4,326 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,38 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# macOS files
+.DS_Store
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# IDE
+.idea/
+\.vscode/
diff --git a/.githooks/pre-commit b/.githooks/pre-commit
@@ -38,14 +38,15 @@ if [ "$CHANGED_FILES" ]; then
   exit 1
 fi
 
+
+echo 'Fixing import order in Python files . . .'
+isort ${PYTHON_FILES[@]}
+
 # Format all staged files, then exit with an error code if any have uncommitted
 # changes.
 echo 'Formatting staged Python files . . .'
 yapf --style .style.yapf -i -r ${PYTHON_FILES[@]}
 
-echo 'Fixing import order in Python files . . .'
-isort ${PYTHON_FILES[@]}
-
 
 CHANGED_FILES=(`git diff --name-only ${PYTHON_FILES[@]}`)
 if [ "$CHANGED_FILES" ]; then

diff --git a/.githooks/pre-push b/.githooks/pre-push
@@ -3,7 +3,7 @@
 # Copyright 2019 Toyota Research Institute.  All rights reserved.
 
 # Git pre-push hook to check commited Python files for pylint issues with
-# driving pylintrc. If there are any pylint errors, it will exit with an error.
+# pylintrc. If there are any pylint errors, it will exit with an error.
 
 BRANCH=$(git rev-parse --abbrev-ref HEAD)
 
@@ -21,7 +21,7 @@ COMMITED_PYTHON_FILES=$(git diff --name-only --diff-filter=ACM --name-only \
 if [[ "$COMMITED_PYTHON_FILES" = "" ]]; then
   echo 'No python files to lint.'
 else
-  pylint $COMMITED_PYTHON_FILES
+  pylint --rcfile=.pylintrc $COMMITED_PYTHON_FILES
   if [ $? -ne 0 ]
   then
     echo "Aborting push due to files with lint."

diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
@@ -0,0 +1,41 @@
+name: Build and publish a Docker image
+
+on:
+  push:
+    branches: ['master', 'release']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.gitignore b/.gitignore
@@ -25,6 +25,9 @@ wheels/
 .installed.cfg
 *.egg
 
+# macOS files
+.DS_Store
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

diff --git a/.pylintrc b/.pylintrc
@@ -3,6 +3,8 @@
 accept-no-param-doc=no
 accept-no-return-doc=yes
 accept-no-yields-doc=yes
+# Ignore protos
+ignore=file_datum_pb2_grpc.py,radar_point_cloud_pb2_grpc.py
 
 [REPORTS]
 reports=no
@@ -55,4 +57,6 @@ disable=
    # allow explicit return in __init__
    E0101,
    # allow catching general exceptions
-   W0703
+   W0703,
+   # allow lazy % or .format() or % formatting in logging functions
+   W1203
diff --git a/.travis.yml b/.travis.yml
@@ -9,8 +9,8 @@ jobs:
       python: 3.6
 # command to install dependencies
 install:
-  - pip install cython==0.29.10
-  - pip install -r dev_requirements.txt
+  - pip install cython==0.29.10 numpy==1.19.4
+  - pip install -r requirements.txt
 # command to run tests
 script:
   - cd $TRAVIS_BUILD_DIR

diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,22 @@
-FROM pytorch/pytorch:1.4-cuda10.1-cudnn7-devel
+FROM nvidia/cuda:11.1-devel-ubuntu18.04
 
 ARG python=3.6
+ENV PYTORCH_VERSION=1.8.1+cu111
+ENV TORCHVISION_VERSION=0.9.1+cu111
 
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 ENV PYTHON_VERSION=${python}
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Set default shell to /bin/bash
+SHELL ["/bin/bash", "-cu"]
+
 RUN apt-get update && apt-get install -y --allow-downgrades --allow-change-held-packages --no-install-recommends \
   build-essential \
   ca-certificates \
+  curl \
   libgl1-mesa-glx \
   libgtk2.0-dev \
   libjpeg-dev \
@@ -22,13 +28,18 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \
   python get-pip.py && \
   rm get-pip.py
 
-# Repo specific dependencies
-# Setup requirements in workspace
+# Install Pytorch
+RUN pip install --no-cache-dir \
+  torch==${PYTORCH_VERSION} \
+  torchvision==${TORCHVISION_VERSION} \
+  -f https://download.pytorch.org/whl/${PYTORCH_VERSION/*+/}/torch_stable.html
+
+# Install python dependencies
 ARG WORKSPACE=/home/dgp
 WORKDIR ${WORKSPACE}
-COPY dev_requirements.txt /tmp/
-RUN pip install cython==0.29.10 numpy==1.16.3
-RUN pip install -r /tmp/dev_requirements.txt --ignore-installed
+COPY requirements.txt /tmp/
+RUN pip install --no-cache-dir cython==0.29.10 numpy==1.19.4
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
 # Settings for S3
 RUN aws configure set default.s3.max_concurrent_requests 100 && \
@@ -37,9 +48,3 @@ RUN aws configure set default.s3.max_concurrent_requests 100 && \
 # Copy workspace and setup PYTHONPATH
 COPY . ${WORKSPACE}
 ENV PYTHONPATH="${WORKSPACE}:$PYTHONPATH"
-
-# Set up streamlit configs
-ARG STREAMLIT_CONFIG_DIR=/root/.streamlit
-RUN mkdir -p ${STREAMLIT_CONFIG_DIR} && \
-    touch ${STREAMLIT_CONFIG_DIR}/credentials.toml && \
-    printf '[general]\nemail = "dummy@domain.com"' > ${STREAMLIT_CONFIG_DIR}/credentials.toml
diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 The MIT License
 
-Copyright (c) 2019-2020 Toyota Research Institute. All rights reserved.
+Copyright (c) 2019-2021 Toyota Research Institute. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2019-2020 Toyota Research Institute.  All rights reserved.
+# Copyright 2019-2021 Toyota Research Institute.  All rights reserved.
 PYTHON ?= python3
 PACKAGE_NAME ?= dgp
 WORKSPACE ?= /home/$(PACKAGE_NAME)
@@ -8,10 +8,16 @@ DOCKER_OPTS ?= \
 	-it \
 	--rm \
 	--shm-size=1G \
+	-e AWS_DEFAULT_REGION \
+	-e AWS_ACCESS_KEY_ID \
+	-e AWS_SECRET_ACCESS_KEY \
 	-e DISPLAY=${DISPLAY} \
 	-v $(PWD):$(WORKSPACE) \
 	-v /var/run/docker.sock:/var/run/docker.sock \
 	-v ~/.ssh:/root/.ssh \
+	-v /data:/data \
+	-v /mnt/fsx:/mnt/fsx \
+	-v ~/.aws:/root/.aws \
 	-v /tmp/.X11-unix/X0:/tmp/.X11-unix/X0 \
 	--net=host --ipc=host
 
@@ -21,6 +27,10 @@ UNITTEST_OPTS ?= --nologcapture -v -s
 
 all: clean test
 
+build-proto:
+	PYTHONPATH=$(PWD):$(PYTHONPATH) \
+	$(PYTHON) setup.py build_py
+
 clean:
 	$(PYTHON) setup.py clean && \
 	rm -rf build dist && \
@@ -31,43 +41,38 @@ clean:
 	find dgp/proto -name "*_pb2.py" | xargs rm -rf
 	find dgp/contribs/pd -name "*_pb2.py" | xargs rm -rf
 
-build-proto:
-	PYTHONPATH=$(PWD):$(PYTHONPATH) \
-	$(PYTHON) setup.py build_py
-
-test: clean build-proto
-	PYTHONPATH=$(PWD):$(PYTHONPATH) \
-	$(UNITTEST) $(UNITTEST_OPTS) $(PWD)/tests/
+develop:
+	pip install cython==0.29.10 numpy==1.19.4 protobuf==3.6.1
+	pip install --editable .
 
 docker-build:
 	docker build \
 	--build-arg WORKSPACE=$(WORKSPACE) \
 	-t $(DOCKER_IMAGE) .
 
+docker-exec:
+	docker exec -it $(DOCKER_IMAGE_NAME) $(COMMAND)
+
+docker-run-tests: build-proto
+	docker run \
+	--name $(DOCKER_IMAGE_NAME)-tests \
+	$(DOCKER_OPTS) $(DOCKER_IMAGE) \
+	$(UNITTEST) $(UNITTTEST_OPTS) $(WORKSPACE)/tests
+
 docker-start-interactive:
-	nvidia-docker run \
+	docker run \
 	$(DOCKER_OPTS) \
 	$(DOCKER_IMAGE) bash
 
-docker-start:
-	nvidia-docker run \
-	-d --name $(DOCKER_IMAGE_NAME) \
-	$(DOCKER_OPTS) $(DOCKER_IMAGE)
-
-docker-exec:
-	nvidia-docker exec -it $(DOCKER_IMAGE_NAME) $(COMMAND)
-
 docker-stop:
 	docker stop $(DOCKER_IMAGE_NAME)
 
-docker-run-tests: build-proto
-	nvidia-docker run \
-	--name $(DOCKER_IMAGE_NAME)-tests \
-	$(DOCKER_OPTS) $(DOCKER_IMAGE) \
-	$(UNITTEST) $(UNITTTEST_OPTS) $(WORKSPACE)/tests
+link-githooks:
+	bash .githooks/link_githooks.sh
 
-docker-start-visualizer:
-	nvidia-docker run \
-	--name $(DOCKER_IMAGE_NAME) \
-	$(DOCKER_OPTS) $(DOCKER_IMAGE) \
-	streamlit run $(WORKSPACE)/dgp/scripts/visualizer.py
+test:
+	PYTHONPATH=$(PWD):$(PYTHONPATH) \
+	$(UNITTEST) $(UNITTEST_OPTS) $(PWD)/tests/
+
+unlink-githooks:
+	unlink .git/hooks/pre-push && unlink .git/hooks/pre-commit
diff --git a/README.md b/README.md
@@ -1,21 +1,31 @@
-[<img src="/docs/tri-logo.jpeg" width="25%">](https://www.tri.global/)
+[<img src="docs/tri-logo.png" width="40%">](https://www.tri.global/)
+
+
+Dataset Governance Policy (DGP)
+=============================
+[![Build Status](https://app.travis-ci.com/TRI-ML/dgp.svg?branch=master)](https://app.travis-ci.com/github/TRI-ML/dgp/builds/238369651)
+[![license](https://img.shields.io/github/license/TRI-ML/dgp.svg)](https://github.com/TRI-ML/dgp/blob/master/LICENSE)
+[![open issues](https://isitmaintained.com/badge/open/TRI-ML/dgp.svg)](https://github.com/TRI-ML/dgp/issues)
 
-TRI Dataset Governance Policy
-==========
 To ensure the traceability, reproducibility and standardization for
-all ML datasets and models generated and consumed within TRI, we developed the
+all ML datasets and models generated and consumed within Toyota Research Institute (TRI), we developed the
 Dataset-Governance-Policy (DGP) that codifies the schema and
 maintenance of all TRI's Autonomous Vehicle (AV) datasets.
 
+<p align="center">
+  <img src="docs/3d-viz-proj.gif" alt="3d-viz-proj"/>
+</p>
 
 ## Components
 - [Schema](dgp/proto/README.md): [Protobuf](https://developers.google.com/protocol-buffers)-based schemas for raw data, annotations
   and dataset management.
 - [DataLoaders](dgp/datasets): Universal PyTorch DatasetClass to load all DGP-compliant datasets.
-- [Visualizer](dgp/scripts/visualizer.py): Simple web-based visualizer for viewing annotations.
-- [CLI](dgp/README.md): Main CLI for handling DGP datasets.
+- [CLI](dgp/README.md): Main CLI for handling DGP datasets and the entrypoint of visulization tools.
+
 
 ## Getting Started
+Please see [getting started](docs/GETTING_STARTED.md) for environment setup.
+
 Getting started is as simple as initializing a dataset-class with the
 relevant dataset JSON, raw data sensor names, annotation types, and
 split information. Below, we show a few examples of initializing a
@@ -40,16 +50,29 @@ directory.
   Started** section above.
 
 ## Build and run tests
-You can build the base docker image and run the tests within docker
+You can build the base docker image and run the tests within [docker container](docs/GETTING_STARTED.md#markdown-header-develop-within-docker)
 via:
 ```sh
 make docker-build
 make docker-run-tests
 ```
 
-## Run Visualizer
-Run streamlit-based interactive visualizer
-via:
-```sh
-make docker-start-visualizer
-```
+## Contributing
+We appreciate all contributions to DGP! To learn more about making a contribution to DGP, please see [contribution page](docs/CONTRIBUTING.md).
+
+## CI Ecosystem
+| Branch | CI | Notes |
+| ---- | ------- | --- |
+| master       | [![Build Status](https://app.travis-ci.com/TRI-ML/dgp.svg?branch=master)](https://app.travis-ci.com/github/TRI-ML/dgp/branches) | master branch build |
+
+
+## 💬 Where to file bug reports
+
+| Type                     | Platforms                                              |
+| - | - |
+| 🚨 **Bug Reports**       | [GitHub Issue Tracker](https://github.com/TRI-ML/dgp/issues) |
+| 🎁 **Feature Requests**  | [GitHub Issue Tracker](https://github.com/TRI-ML/dgp/issues) |
+
+## 👩‍💻  The Team 👨‍💻
+
+DGP is developed and currently maintained by *Quincy Chen, Arjun Bhargava, Chao Fang, Chris Ochoa and Kuan-Hui Lee* from ML-Engineering team at [Toyota Research Institute (TRI)](https://www.tri.global/), with contributions coming from ML-Research team at TRI, [Woven Planet](https://www.woven-planet.global/en) and [Parallel Domain](https://paralleldomain.com/).