Merge pull request #186 from Riverscapes/dev

January release redo

.dockerignore

@@ -0,0 +1,30 @@
+**/__pycache__
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+
+**/docs
+.github
+.venv
+PUBLIC
+Workspaces

.github/workflows/docs_publish.yml

@@ -22,21 +22,35 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
 
-      # Runs a single command using the runners shell
-      - name: Run a one-line script
-        run: echo Hello, world!
-
-      # Runs a set of commands using the runners shell
-      - name: Run a multi-line script
-        run: |
-          echo Add other actions to build,
-          echo test, and deploy your project.
-
-
-      # Use GitHub Actions' cache to shorten build times and decrease load on servers
+        # Use GitHub Actions' cache to shorten build times and decrease load on servers
       - uses: actions/cache@v1
         with:
           path: vendor/bundle
           key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }}
           restore-keys: |
-            ${{ runner.os }}-gems-          
+            ${{ runner.os }}-gems-
+
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+
+
+      # Build each site
+      - name: Bundle install dependencies
+        run: sh scripts/docs-bundler.sh
+
+      - name: Publish sites
+        run: sh scripts/docs-publish.sh
+
+        # Custom Jekyll
+      - name: Deploy to GitHub Pages
+        if: success()
+        uses: crazy-max/ghaction-github-pages@v2
+        with:
+          target_branch: gh-pages
+          build_dir: PUBLIC
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+

.gitignore

@@ -5,7 +5,8 @@ __pycache__/
 
 # PUBLIC is where the GH pages branch gets built
 PUBLIC
-
+# Ruby gems installed using bundler
+.vendor
 # C extensions
 *.so
 _site
@@ -105,6 +106,7 @@ celerybeat.pid
 
 # Environments
 .env
+.env.*
 .venv
 env/
 venv/

Dockerfile

@@ -0,0 +1,126 @@
+############################################################################
+# RSTools Dockerbox
+# Author: Matt Reimer
+# Description: A working linux box that can run our riverscapes-tools
+############################################################################
+
+# We use OSGEO's ubuntu box as a base so we don't need to compile GDAL ourselves
+FROM osgeo/gdal:ubuntu-full-3.1.2 AS WORKER
+ARG DEBIAN_FRONTEND=noninteractive
+ARG CACHEBUST=1
+RUN apt-get update
+
+# Installing Prerequisite Packages
+# =======================================================================================
+RUN apt-get update && apt-get install vim git awscli curl locales -y
+
+# Our python scripts use UTF-8 so let's make sure we're in the right locale
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+
+################################################################################################
+# TauDEM as a multi-stage install: DO all our compiling on a multi-stage build
+# Method borrowed from: https://github.com/NOAA-OWP/cahaba/blob/dev/Dockerfile.prod
+# https://hub.docker.com/r/osgeo/gdal/tags?page=1&ordering=last_updated&name=ubuntu-full-3.2.1
+################################################################################################
+
+FROM WORKER AS BUILDER
+WORKDIR /opt/builder
+ARG dataDir=/data
+ARG projectDir=/foss_fim
+ARG depDir=/dependencies
+ARG taudemVersion=bf9417172225a9ce2462f11138c72c569c253a1a
+ARG taudemVersion2=81f7a07cdd3721617a30ee4e087804fddbcffa88
+ENV DEBIAN_FRONTEND noninteractive
+ENV taudemDir=$depDir/taudem/bin
+ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
+
+RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/* 
+
+RUN git clone https://github.com/dtarb/taudem.git
+RUN git clone https://github.com/fernandoa123/cybergis-toolkit.git taudem_accelerated_flowDirections
+
+RUN apt-get update && apt-get install -y cmake mpich \
+    libgtest-dev libboost-test-dev libnetcdf-dev && rm -rf /var/lib/apt/lists/* 
+
+## Compile Main taudem repo ##
+RUN mkdir -p taudem/bin
+RUN cd taudem \
+    && git checkout $taudemVersion \
+    && cd src \
+    && make
+
+## Compile taudem repo with accelerated flow directions ##
+RUN cd taudem_accelerated_flowDirections/taudem \
+    && git checkout $taudemVersion2 \
+    && mkdir build \
+    && cd build \
+    && cmake .. \
+    && make
+
+RUN mkdir -p $taudemDir
+RUN mkdir -p $taudemDir2
+
+## Move needed binaries to the next stage of the image
+RUN cd taudem/bin && mv -t $taudemDir flowdircond aread8 threshold streamnet gagewatershed catchhydrogeo dinfdistdown pitremove
+RUN cd taudem_accelerated_flowDirections/taudem/build/bin && mv -t $taudemDir2 d8flowdir dinfflowdir
+
+
+################################################################################################
+# Now switch back to the worker and Copy the builder's scripts over
+################################################################################################
+FROM WORKER
+
+ARG depDir=/dependencies
+ARG taudemDir=$depDir/taudem/bin
+ARG taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
+RUN mkdir -p $depDir
+COPY --from=builder $depDir $depDir
+
+# Add TauDEM to the path so we have access to those methods
+ENV PATH="${taudemDir}:${taudemDir2}:${PATH}"
+
+# Let's try adding gdal this way
+# RUN add-apt-repository ppa:ubuntugis/ppa && apt-get update
+RUN apt update --fix-missing
+RUN apt install -y p7zip-full python3-pip time mpich=3.3.2-2build1 parallel=20161222-1.1 libgeos-dev=3.8.0-1build1 expect=5.45.4-2build1 libspatialindex-dev
+RUN apt auto-remove
+
+# https://launchpad.net/~ubuntugis/+archive/ppa/
+# Install node please. Order matters here so it should come after all other apt-get update steps or it may be removed by a cleanup process
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get install -y nodejs
+RUN npm install -g npm
+RUN node --version
+RUN npm --version
+
+# Add in the right version of pip and modules
+# =======================================================================================
+RUN python3 -m pip install --upgrade pip 
+RUN python3 -m pip install virtualenv
+
+# Now install riverscapes CLI
+ARG CACHEBREAKER3=121
+RUN npm -v
+RUN npm install -g @riverscapes/cli
+
+# Never run as root
+ARG GroupName=nar
+RUN addgroup $GroupName
+
+## Set user:group for running docker in detached mode
+USER root:$GroupName
+
+
+# Pull in the python libraries. We build it in a different stage to keep the id_rsa private
+# =======================================================================================
+COPY . /usr/local/riverscapes-tools
+WORKDIR /usr/local/riverscapes-tools
+RUN ./scripts/bootstrap.sh
+
+WORKDIR /shared
+
+ENTRYPOINT [ "sh", "/usr/local/riverscapes-tools/bin/run.sh"]

README.md

@@ -13,4 +13,14 @@ Both `./lib` and `./packages` contain pep8-compliant python packages.
 * `./lib` is for anything that can be shared between tools
 * `./packages` is for the tools themselves. They cannot depend on one another. 
 
-If you find that your tools have code they need to share you can either pull it into the `./lib/commons` package or, if there's enough code, create a new `./lib/whatever` package.
+If you find that your tools have code they need to share you can either pull it into the `./lib/commons` package or, if there's enough code, create a new `./lib/whatever` package.
+
+## Repo guidelines:
+
+We're pretty much using the [Gitflow Workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow)
+
+1. All work ***MUST*** happen on branches. 
+2. Make sure your pull requests merge onto `dev` and not `master`.
+3. Hotfixes may be applied directly to `master`... carefully and after lots of testing.
+4. Add your issue and/or pull request to a milestone to make sure it will be included.
+5. Attach your commits to issues and pull requests by [referencing them in commit messages](https://docs.github.com/en/enterprise/2.16/user/github/managing-your-work-on-github/closing-issues-using-keywords)

bin/requirements.txt

@@ -0,0 +1,55 @@
+affine==2.3.0
+appdirs==1.4.4
+astroid==2.4.2
+attrs==20.1.0
+autopep8==1.5.4
+awscli==1.18.97
+botocore==1.17.20
+click==7.1.2
+click-plugins==1.1.1
+cligj==0.5.0
+colorama==0.4.3
+Cython==0.29.7
+decorator==4.4.2
+distlib==0.3.1
+docutils==0.15.2
+filelock==3.0.12
+GDAL>=3.0
+html5print==0.1.2
+isort==5.4.2
+Jinja2==2.11.2
+jmespath==0.10.0
+lazy-object-proxy==1.4.3
+matplotlib==3.3.1
+mccabe==0.6.1
+networkx==2.5
+numpy==1.19.0
+postgis==1.0.4
+prompt-toolkit==1.0.14
+psycopg2-binary==2.8.5
+pyasn1==0.4.8
+pycodestyle==2.6.0
+pygeoprocessing==2.1.0
+Pygments==2.6.1
+PyInquirer==1.0.3
+pylint==2.6.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+PyYAML==5.3.1
+rasterio==1.1.5
+regex==2020.7.14
+rsa==4.5
+Rtree==0.9.4
+s3transfer==0.3.3
+scikit-fuzzy==0.4.2
+scipy==1.5.1
+semver==2.10.2
+Shapely==1.7.1
+six==1.15.0
+snuggs==1.4.7
+termcolor==1.1.0
+toml==0.10.1
+urllib3==1.25.9
+virtualenv==20.0.26
+wcwidth==0.2.5
+wrapt==1.12.1

bin/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eu
+IFS=$'\n\t'
+
+echo "$SHELL_SCRIPT" > /usr/local/runner.sh
+chmod +x /usr/local/runner.sh
+
+( exec "/usr/local/runner.sh" )

docs/CNAME

@@ -0,0 +1 @@
+tools.riverscapes.xyz

docs/_config.yml

@@ -26,6 +26,7 @@ defaults:
 
 # Files/Folders to exclude from publishing
 exclude:
+  - vendor
   - src
   - LICENSE
   - README.md

docs/developing/Docker.md

@@ -0,0 +1,12 @@
+---
+title: Running with Docker
+---
+
+This repo comes with a Dockerfile that you should be able to use to run these tools without any prerequisites (except docker of course)
+
+### building
+
+1. Run `./scripts/dockerBuild.sh`
+2. Make a script to run (i.e. `./scripts/testscript.sh`)
+3. Create an `.env`
+4. `./scripts/dockerRun.sh ~/Work/data/bratDockerShare ./scripts/testscript.sh `

docs/developing/Release_Checklist.md

@@ -0,0 +1,17 @@
+---
+title: Release Checklist
+---
+
+We're moving to monthly releases.
+
+1. [ ] resolve any `branch` -> `dev` pull requests
+1. [ ] Increment version numbers if necessary
+1. [ ] Create a pull request to merge dev --> master
+1. [ ] Run every tool once through cybercastor staging on `dev`
+1. [ ] Quick code review
+1. [ ] Merge the PR `dev` --> `master`
+
+-----------
+
+Out of order:
+

docs/index.md

@@ -6,4 +6,6 @@ Tools:
 
 * [VBET]({{ site.baseurl }}/vbet)
 * [RSContext]({{ site.baseurl }}/rscontext)
-* [BRAT]({{ site.baseurl }}/brat)
+* [BRAT]({{ site.baseurl }}/brat)
+* [RVD]({{ site.baseurl }}/rvd)
+* [GNAT]({{ site.baseurl }}/gnat)

lib/commons/rscommons/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "1.1.0"