cromshell

#!/usr/bin/env bash

################################################################################

# Setup variables for the script:
UNALIASED_SCRIPT_NAME=$( python -c "import os;print (os.path.realpath(\"${BASH_SOURCE[0]}\"))" )
SCRIPTDIR="$( cd "$( dirname "${UNALIASED_SCRIPT_NAME}" )" && pwd )"
SCRIPTNAME=$( echo $0 | sed 's#.*/##g' )
MINARGS=1
MAXARGS=99
PREREQUISITES="curl jq mail column tput rev"

# Determine if this shell is interactive:
ISINTERACTIVESHELL=true
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && ISINTERACTIVESHELL=false

# Required for the aliased call to checkPipeStatus:
shopt -s expand_aliases

################################################################################
COLOR_NORM='\033[0m'
COLOR_UNDERLINED='\033[1;4m'
COLOR_FAILED='\033[1;37;41m'
COLOR_SUCCEEDED='\033[1;30;42m'
COLOR_RUNNING='\033[0;30;46m'

################################################################################

# Setup Cromshell config details:
CROMSHELL_CONFIG_DIR=${HOME}/.cromshell
mkdir -p ${CROMSHELL_CONFIG_DIR}
CROMWELL_SUBMISSIONS_FILE="${CROMSHELL_CONFIG_DIR}/all.workflow.database.tsv"
[ ! -f ${CROMWELL_SUBMISSIONS_FILE} ] && echo -e "DATE\tCROMWELL_SERVER\tRUN_ID\tWDL_NAME\tSTATUS" > ${CROMWELL_SUBMISSIONS_FILE} 

# Find the CROMWELL Server:
CROMWELL_SERVER_FILE=${CROMSHELL_CONFIG_DIR}/cromwell_server.config
CROMWELL_NEEDS_SETUP=true
[ -f ${CROMWELL_SERVER_FILE} ] && CROMWELL_NEEDS_SETUP=false
if ! $CROMWELL_NEEDS_SETUP ; then
  # NOTE: Server file allows for line comments using the prefix `#`
  CROMWELL_SERVER_FROM_FILE=$( grep -v '^#' ${CROMWELL_SERVER_FILE} | head -n1 )
fi

# read env var, or use default server URL
export CROMWELL_URL=${CROMWELL_URL:-"${CROMWELL_SERVER_FROM_FILE}"}
FOLDER_URL=$( echo ${CROMWELL_URL} | sed -e 's#ht.*://##g' ) 
CROMWELL_METADATA_PARAMETERS="excludeKey=submittedFiles&expandSubWorkflows=true"

CURL_CONNECT_TIMEOUT=5
let CURL_MAX_TIMEOUT=2*${CURL_CONNECT_TIMEOUT}

PING_CMD='ping -c1 -W1 -w10'
if [[ $( uname ) == "Darwin" ]] ; then
  PING_CMD='ping -c1 -W1000 -t10'
fi

################################################################################

SUB_COMMAND=''
WORKFLOW_ID_LIST=''
WORKFLOW_ID=''
WORKFLOW_SERVER_URL=''
OPTIONS=''

TERMINAL_STATES="Succeeded Failed Aborted"

################################################################################

function turtle() 
{
  if [[ -z $1 ]]; then
    eye="'"
  else
    eye="x"
  fi
  error "                  __     "
  error "       .,-;-;-,. /${eye}_\\    "
  error "     _/_/_/_|_\\_\\) /     "
  error "   '-<_><_><_><_>=/\\     "
  error "     \`/_/====/_/-'\\_\\    " 
  error "      \"\"     \"\"    \"\"    " 
}

function turtleDead()
{
  error "${COLOR_FAILED}    ,,    ,,     ,,      ${COLOR_NORM}"
  error "${COLOR_FAILED}    \\‾\\,-/‾/====/‾/,     ${COLOR_NORM}"
  error "${COLOR_FAILED}     \\/=<‾><‾><‾><‾>-,   ${COLOR_NORM}"
  error "${COLOR_FAILED}     / (\\‾\\‾|‾/‾/‾/‾     ${COLOR_NORM}"
  error "${COLOR_FAILED}    \\‾x/ ˙'-;-;-'˙       ${COLOR_NORM}"
  error "${COLOR_FAILED}     ‾‾                  ${COLOR_NORM}"
}

################################################################################

function simpleUsage()
{
  if [[ $# -ne 0 ]] ; then
    usage | grep "  ${1} " | sed -e 's#\(.*]\).*#\1#g' -e "s#^[ \\t]*#Usage:    ${SCRIPTNAME} #g"
  else
    echo -e "Usage:    ${SCRIPTNAME} [-t TIMEOUT] SUB-COMMAND [options]"
    echo -e "Run and inspect workflows on a Cromwell server."
  fi
}

#Define a usage function:
function usage() 
{
  simpleUsage
  echo -e ""
  echo -e "CROMWELL_URL=$CROMWELL_URL"
  echo -e ""
  echo -e "If no workflow-id is specified then the last submitted workflow-id is assumed."
  echo -e "Alternatively, negative numbers can be used to refer to previous workflows."
  echo -e ""
  echo -e "  example usage:"
  echo -e "      cromshell submit workflow.wdl inputs.json options.json dependencies.zip"
  echo -e "      cromshell status"
  echo -e "      cromshell -t 50 status"
  echo -e "      cromshell logs -2"
  echo -e ""
  echo -e "Supported Flags:"
  echo -e "  -t TIMEOUT             Set the curl connect timeout to TIMEOUT seconds."
  echo -e "                         Also sets the curl max timeout to 2*TIMEOUT seconds."
  echo -e "                         TIMEOUT must be an integer."
  echo -e ""
  echo -e "Supported Subcommands:"
  echo -e ""
  echo -e "  Start/Stop workflows:"
  echo -e "   submit [-w] <wdl> <inputs_json> [options_json] [included_wdl_zip_file] Submit a new workflow"
  echo -e "        -w                     Wait for workflow to transition from 'Submitted' to some other status"
  echo -e "                               before ${SCRIPTNAME} exits"
  echo -e "        included_wdl_zip_file  Zip file containing any WDL files included in the input WDL"
  echo -e ""
  echo -e "   abort [workflow-id] [[workflow-id]...]                            Abort a running workflow."
  echo -e
  echo -e "  Query workflow status:"
  echo -e "   status [workflow-id] [[workflow-id]...]                           Check the status of a workflow."
  echo -e "   metadata [workflow-id] [[workflow-id]...]                         Get the full metadata of a workflow."
  echo -e "   slim-metadata [workflow-id] [[workflow-id]...]                    Get a subset of the metadata from a workflow."
  echo -e "   execution-status-count [workflow-id] [[workflow-id]...]           Get the summarized status of all jobs in the workflow."
  echo -e "   timing [workflow-id] [[workflow-id]...]                           Open the timing diagram in a browser."
  echo -e
  echo -e "  Logs:"
  echo -e "   logs [workflow-id] [[workflow-id]...]                             List the log files produced by a workflow."
  echo -e "   fetch-logs [workflow-id] [[workflow-id]...]                       Download all logs produced by a workflow."
  echo -e
  echo -e "  Job Outputs:"
  echo -e "   list-outputs [workflow-id] [[workflow-id]...]                     List all output files produced by a workflow."
  echo -e "   fetch-all [workflow-id] [[workflow-id]...]                        Download all output files produced by a workflow."
  echo -e
  echo -e "  Get email notification on job completion:"
  echo -e "   notify [workflow-id] [daemon-server] email [cromwell-server]"
  echo -e "        email             Email address to which to send the notification."
  echo -e "        daemon-server     Server on which to run the notification daemon."
  echo -e ""
  echo -e "  Display a list jobs submitted through cromshell:"
  echo -e "   list [-c] [-u]                                            "
  echo -e "         -c               Color the output by completion status."
  echo -e "         -u               Check completion status of all unfinished jobs."
  echo -e "" 
  echo -e "  Clean up local cached list:"
  echo -e "   cleanup [-s STATUS]    Remove completed jobs from local list."
  echo -e "                          Will remove all jobs from the local list that are in a completed state,"
  echo -e "                          where a completed state is one of: $(echo ${TERMINAL_STATES} | tr ' ' ',' )" 
  echo -e "         -s STATUS        If provided, will only remove jobs with the given STATUS from the local list."
  echo -e ""
  echo -e "Return values:"
  echo -e "  0                  SUCCESS"
  echo -e "  ANYTHING_BUT_ZERO  FAILURE/ERROR"
  echo
}

#Display a message to std error:
function error() 
{
  echo -e "$1" 1>&2 
}

# Make a better temp file that gets cleaned up on script exit:
TMPFILELIST=''
function makeTemp()
{
  local f
  f=$( mktemp )
  TMPFILELIST="${TMPFILELIST} $f"
  echo $f
}

# Function to clean our temp files:
function cleanTempVars()
{
  rm -f ${TMPFILELIST}
}

# Function to be called at exit:
function at_exit()
{
  cleanTempVars
}
# Make sure we clean up on exit:
trap at_exit EXIT 

# Print the name and value of a simple variable:
function printVar()
{
  error "$1 = ${!1}"
}

# Checks the bash built-in PIPESTATUS variable for any failures
# If given strings will output the string corresponding to the failure
# position in PIPESTATUS of any failures in the chain of commands 
# that occurred.
# This function should be called as `checkPipeStatus` as per the 
# alias below it.
function _checkPipeStatus()
{
  local hadFailure=false

  for (( i = 0 ; i < ${#RC[@]} ; i++ )) ; do 
    st=${RC[i]}
    if [ $st -ne 0 ] ; then
      # If we were passed a description string for this error in the pipe, then we print it:
      let argIndex=$i+1
      description=${!argIndex}
      [[ ${#description} -ne 0 ]] && error "$description"
      hadFailure=true  
    fi
  done

  if $hadFailure ; then
    return 10
  fi
  return 0
}
alias checkPipeStatus='RC=( "${PIPESTATUS[@]}" );_checkPipeStatus'

function checkPrerequisites
{
  local missing=""
  local foundMissing=false
  for c in ${@} ; do
    which $c &> /dev/null
    r=$?
    [[ $r -ne 0 ]] && foundMissing=true && missing="${missing}$c "
  done

  if $foundMissing ; then
    error "Error: the following commands could not be found:"
    error "  $missing"
    error "Please install them and try again."
    if [[ $(uname) == 'Darwin' ]] ; then
      error "NOTE: You likely must use homebrew to install them."
    fi
    return 1
  fi
  return 0
}

function checkForComplexHelpArgument() 
{
  for arg in $@ ; do
    case $arg in 
      -h|--h|-\?|--\?|--help|-help|help) usage; exit 0;;
    esac
  done
}

################################################################################
# Cromshell-specific Utility Functions:

# Handle arguments just expecting workflow IDs:
# Will populate the $WORKFLOW_ID_LIST global variable or exit.
function handleArgs_workflow_ids()
{
  #Read args:
  if [ $# -eq 0 ] ; then
    # No specified workflow ID.
    # Get the last one.
    populateLastWorkflowId
    WORKFLOW_ID_LIST="${WORKFLOW_ID}"
  else
    while [ $# -gt 0 ] ; do
      populateWorkflowIdAndServerUrl ${1}
      WORKFLOW_ID_LIST="${WORKFLOW_ID_LIST} ${1}"

      #Get next argument in $1:
      shift
    done
  fi
}

function matchWorkflowId() 
{
  local rv=1

  if [[ "${1}" =~ ^[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$ ]] ; then
    rv=0
  fi

  echo ${rv}
  return ${rv}
}

function invalidSubCommand()
{
  error "${SCRIPTNAME} $1: invalid $2: $3"
  error ""
  if [[ ${#1} -eq 0 ]] ; then
    simpleUsage
  else
    simpleUsage $1
  fi
  echo "Try \`$SCRIPTNAME -h' for more information."
  exit 3;
}

function getAnswerFromUser()
{
  local prompt="${1}" 
  local acceptableValues="${2}"
  local responseVar="${3}" 

  local haveGoodValue=false
  while ! $haveGoodValue ; do
    read -p "${prompt} [$( echo ${acceptableValues} | tr ' ' '/' )] : " ${responseVar}

    for okVal in ${acceptableValues} ; do
      if [[ "$( echo ${!responseVar} | tr a-z A-Z)" == "$( echo ${okVal} | tr a-z A-Z)" ]] ; then
        haveGoodValue=true
      fi
    done

    ! $haveGoodValue && error "Please enter one of the following: $( echo ${acceptableValues} | tr ' ' '/' )" && error ""
  done
}

function assertCanCommunicateWithServer
{
  # Make sure we can talk to the cromwell server:
  serverName=$( echo ${1}| sed 's#.*://##g'| sed 's#:[0-9]*##g' )
  $PING_CMD $serverName &> /dev/null
  r=$?
  if [[ $r -ne 0 ]] ; then 
    turtleDead 
    error "Error: Cannot communicate with Cromwell server: $serverName" 
    exit 4
  fi
}

function assertHavePreviouslySubmittedAJob()
{
  local nLines=$( wc -l ${CROMWELL_SUBMISSIONS_FILE} | awk '{print $1}' )
  if [[ $nLines -eq 1 ]] ; then
    error "Error: You have never submitted any jobs using ${SCRIPTNAME}."
    error "       You must enter a workflow ID to get information on it."
    exit 99
  fi
}

function populateLastWorkflowId() 
{
  assertHavePreviouslySubmittedAJob
  WORKFLOW_ID=$( tail -n1 ${CROMWELL_SUBMISSIONS_FILE} | awk '{print $3}' )
}

function populateWorkflowIdAndServerUrl()
{
  local userSpecifiedId=$1

  # If the user specified a negative number, get the nth last workflow:
  if [[ $userSpecifiedId =~ ^-[[:digit:]]+$ ]]; then
    local row=${userSpecifiedId#-}
    assertHavePreviouslySubmittedAJob
    WORKFLOW_ID=$(tail -n $row $CROMWELL_SUBMISSIONS_FILE | head -n 1 | awk '{print $3}' )
    WORKFLOW_SERVER_URL=$(tail -n $row $CROMWELL_SUBMISSIONS_FILE | head -n 1 | awk '{print $2}' )
  else
    # If the user specified anything at this point, assume that it 
    # is a workflow UUID:
    if [ -n "$userSpecifiedId" ]; then 
      WORKFLOW_ID=$userSpecifiedId 
      WORKFLOW_SERVER_URL=${CROMWELL_URL}

      # Attempt to get the workflow server from the submission database:  
      local tmpFile=$( makeTemp )
      grep ${WORKFLOW_ID} ${CROMWELL_SUBMISSIONS_FILE} > ${tmpFile}
      r=$?
      [ $r -eq 0 ] && WORKFLOW_SERVER_URL=$( awk '{print $2}' ${tmpFile} )

      # If the user specified nothing, get the last workflow ID:  
    else
      populateLastWorkflowId
      WORKFLOW_SERVER_URL=$(tail -n1 $CROMWELL_SUBMISSIONS_FILE | awk '{print $2}' )
    fi
  fi

  # Validate our workflow ID:
  if [[ $( matchWorkflowId ${WORKFLOW_ID} ) -ne 0 ]] ; then
    if [ -n "$userSpecifiedId" ] ; then
      error "Invalid workflow ID: ${1}"
      error "It is likely that there are not this many records in your cromshell file."
    else
      error "Invalid workflow ID: ${WORKFLOW_ID}"
    fi
    exit 5
  fi
}

function _turtleSpinner()
{
  # Require a flag file to be given to this function.
  # This keeps it safe.
  if [[ $# -ne 1 ]] ; then
    return 1
  fi

  local flagFile=$1

  # Let's be fancy and make turtles that are walking while we wait:
  local tmpTurtFile=$( makeTemp )
  local turtFile=$( makeTemp )
  local turtRFile=$( makeTemp )
  turtle &> ${tmpTurtFile}
  sed 's#^  ##g' ${tmpTurtFile} > ${turtFile}
  rev ${turtFile} | sed -e 's$/$#$g' -e 's#\\#/#g' -e 's$#$\\$g' -e 's#(#)#g' -e 's#)#(#g' > ${turtRFile}

  # Set up some loop variables:
  local i=3
  local incrementString="+1"
  local curTurtFile=${turtFile}

  # Let's loop while the ${flagFile} exists.
  # Note that this will be spun off into its own thread:
  while [ -f ${flagFile} ] ; do
    # Create padding for turtle:
    padding=$( printf "%${i}s" "" )

    # Display our turtle, moved a little to the right:
    cat ${curTurtFile} | sed -e "s#^#${padding}#g"
    sleep 0.1

    # Move the cursor back up to the top line of the turtle
    # so we create the illusion of movement:
    tput cuu 6

    # Now we update our loop variables in a clever way
    # so that the turtle will go back and forth.
    let i=i${incrementString}
    if [ $i -eq 15 ] ; then 
      incrementString="-1" 
      curTurtFile=${turtRFile}
    elif [ $i -eq 0 ] ; then
      incrementString="+1" 
      curTurtFile=${turtFile}
    fi

  done & # The & here spins this off as a background thread/process
}

function waitOnSubmittedStatus() 
{
  local id=$1
  local cromwellServerUrl=$2

  echo "Waiting on status to progress from 'Submitted' ..."
  echo

  # Use tput to save the cursor position:
  tput sc

  # The turtle starts 9 rows above where the cursor is at this point,
  # so if we need to move it, we have to move the cursor there before
  # we output anything:
  tput cuu 9

  # Reserve the name for a file whose non-existence will cause 
  # the following loop to exit.
  # This is good because all temp files made with `makeTemp` 
  # are automatically removed at exit, so this sub-process
  # will always die gracefully.
  local flagFile=$(makeTemp)
  echo 'TEST' > ${flagFile}

  # Kick off a turtle spinner...
  _turtleSpinner ${flagFile}

  # Now we check the status periodically and if we get a new one
  # Other than `Submitted` we can tell the other process to finish.
  local gotNewStatus=false
  local isDone=false
  local nChecks=0
  local statusFile=$( makeTemp )
  local r
  while ! $isDone ; do
    # Wait for a little bit so we don't kill the server:
    sleep 2

    # Check the status of our last submission:
    status ${id} ${cromwellServerUrl} &> ${statusFile}
    r=$?  
    grep -q '^[ \t]*"status": "Submitted",' ${statusFile}

    # We could no longer field the status as `Submitted`, so we're done.
    if [ $? -eq 1 ] && [ $r -ne 2 ] ; then
      gotNewStatus=true
    fi

    # Check if our status changed yet:   
    if $gotNewStatus ; then
      isDone=true
 
      # Delete the flag file to end the display process:
      rm -f ${flagFile}

      # Wait for the _turtleSpinner to exit:
      wait

      # Clear out the lines from our fancy animation:
      tput el;echo;tput el;echo;tput el;echo;tput el;echo;tput el;echo;tput el;echo;tput el;echo;tput el;echo
      tput cuu 8

      # Display the status file:
      cat ${statusFile} | sed 's#^Using ##g'
  
    # If we have waited too long, then we quit:
    elif [[ $nChecks -ge 10 ]] ; then
      isDone=true
      
      # Delete the flag file to end the display process:
      rm -f ${flagFile}
      
      # Wait for the _turtleSpinner to exit:
      wait

      echo "{\"id\":\"${id}\",\"status\":\"Submitted\"}"
      error "WARNING: Could not validate submission status with server.  Try again later."
      break
    fi

    let nChecks=$nChecks+1
  done

  # Restore cursor position:
  tput rc
  
  # Add a line for padding:
  echo
  
  # Send a proper return code:
  if $gotNewStatus ; then
    return 0
  else
    return 1
  fi
}

# Submit a workflow and arguments to the Cromwell Server
function submit() 
{
  local doWait=false
    
  # Handle Arguments:
  local OPTIND
  while getopts "w" opt ; do
    case ${opt} in 
      w)
        doWait=true
        ;;
      *)
        invalidSubCommand submit flag ${OPTARG}
        ;; 
    esac 
  done
  shift $((OPTIND-1)) 
 
  # ---------------------------------------- 

  assertCanCommunicateWithServer $CROMWELL_URL

  local response=$(curl --connect-timeout $CURL_CONNECT_TIMEOUT --max-time $CURL_MAX_TIMEOUT -s -F workflowSource=@${1}  ${2:+ -F workflowInputs=@${2}} ${3:+ -F workflowOptions=@${3}} ${4:+ -F workflowDependencies=@${4}} ${CROMWELL_URL}/api/workflows/v1)
  local r=$?

  # Check to make sure that we actually submitted the job correctly 
  # and the server ate it well:
   
  # Did the Curl call fail?
  [ $r -ne 0 ] && error "FAILED TO SUBMIT JOB" && return $r

  # Get our status and job ID:
  local st=$( echo "${response}" | jq -r '.status' 2>/dev/null )
  local id=$( echo "${response}" | jq -r '.id' 2>/dev/null )

  # If the status is not `Submitted`, something went wrong:
  if [[ "${st}" != "Submitted" ]] ; then
    turtleDead
    error
    error "Error: Server reports that job was not properly submitted.  Server response:"
    error "${response}"
    exit 9
  fi

  # If the ID is not an ID, something went wrong:
  if [[ $( matchWorkflowId ${id} ) -ne 0 ]] ; then
    turtleDead
    error
    error "Error: Did not get a valid ID back.  Something went wrong.  Server response:"
    error "${response}"
    exit 9
  fi

  # If we get here, we successfully submitted the job and should track it locally:
  turtle
  echo "${response}" 
  
  local runDir=${CROMSHELL_CONFIG_DIR}/${FOLDER_URL}/${id}
  mkdir -p ${runDir}

  cp ${1} ${2} ${3} ${4} ${runDir}/

  echo -e "$(date +%Y%m%d_%H%M%S)\t${CROMWELL_URL}\t${id}\t$(basename ${1})\tSubmitted" >> ${CROMWELL_SUBMISSIONS_FILE}

  # Now that we've submitted our task, we should see if we have to wait:
  if $doWait ; then
    waitOnSubmittedStatus ${id} ${CROMWELL_URL}
  fi

  return 0
}

# Check the status of a Cromwell job UUID
function status()
{
  local retVal=0

  assertCanCommunicateWithServer $2
  local f=$( makeTemp )
  curl --connect-timeout $CURL_CONNECT_TIMEOUT --max-time $CURL_MAX_TIMEOUT -s ${2}/api/workflows/v1/${1}/status > $f
  [[ $? -ne 0 ]] && error "Could not connect to Cromwell server." && return 2

  grep -qE '"Failed"|"Aborted"|"fail"' $f
  r=$?
  [[ $r -eq 0 ]] && retVal=1

  if [[ $retVal -eq 1 ]]; then
    turtleDead
  else
    turtle
  fi

  cat $f | jq . 
  checkPipeStatus "Could not read tmp file JSON data." "Could not parse JSON output from cromwell server."

  # Update ${CROMWELL_SUBMISSIONS_FILE}:
  local st=$( cat $f | jq . | grep status | sed -e 's#.*: ##g' | tr -d '",' )
  sed -i .bak -e "s#\\(.*${1}.*\\.wdl\\)\\t*.*#\\1$(printf '\t')${st}#g" ${CROMWELL_SUBMISSIONS_FILE}

  return $retVal
}

# Get the logs of a Cromwell job UUID
function logs() 
{
  assertCanCommunicateWithServer $2
  turtle
  curl --connect-timeout $CURL_CONNECT_TIMEOUT --max-time $CURL_MAX_TIMEOUT -s ${2}/api/workflows/v1/${1}/logs | jq . 
  checkPipeStatus "Could not connect to Cromwell server." "Could not parse JSON output from cromwell server."
  return $?
}

# Get the metadata for a Cromwell job UUID
function metadata() 
{
  assertCanCommunicateWithServer $2
  turtle
  curl --connect-timeout $CURL_CONNECT_TIMEOUT --max-time $CURL_MAX_TIMEOUT --compressed -s ${2}/api/workflows/v1/${1}/metadata?${CROMWELL_METADATA_PARAMETERS} | jq . 
  checkPipeStatus "Could not connect to Cromwell server." "Could not parse JSON output from cromwell server."
  return $?
}

# Get the metadata in condensed form for a Cromwell job UUID
function slim-metadata() 
{
  assertCanCommunicateWithServer $2
  turtle
  curl --connect-timeout $CURL_CONNECT_TIMEOUT --max-time $CURL_MAX_TIMEOUT --compressed -s "${2}/api/workflows/v1/$1/metadata?includeKey=executionStatus&includeKey=backendStatus&expandSubWorkflows=true" | jq .
  checkPipeStatus "Could not connect to Cromwell server." "Could not parse JSON output from cromwell server."
  return $?
}

# Get the status of the given job and how many times it was run
function execution-status-count() 
{
  assertCanCommunicateWithServer $2
  turtle
  f=$( makeTemp )
  curl --connect-timeout $CURL_CONNECT_TIMEOUT --max-time $CURL_MAX_TIMEOUT --compressed -s ${2}/api/workflows/v1/$1/metadata?${CROMWELL_METADATA_PARAMETERS} > $f
  [[ $? -ne 0 ]] && error "Could not connect to Cromwell server." && return 9

  # Make sure the query succeeded:
  if [[ "$( cat $f | jq '.status' | sed -e 's#^"##g' -e 's#"$##g' )" == 'fail' ]] ; then
    reason=$( cat $f | jq '.message' | sed -e 's#^"##g' -e 's#"$##g' )
    error "Error: Query to cromwell server failed: ${reason}"
    return 11
  fi

  # Make it pretty for the user:
  cat $f | jq '.calls | to_entries | map({(.key): .value | flatten | group_by(.executionStatus) | map({(.[0].executionStatus): . | length}) | add})'
  checkPipeStatus "Could not read tmp file JSON data." "Could not parse JSON output from cromwell server."
  return $?
}

# Bring up a browser window to view timing information on the job.
function timing() 
{
  turtle
  echo "Opening timing information in a web browser for job ID: ${1}"
  open ${2}/api/workflows/v1/${1}/timing
  return $?
}

# Cancel a running job.
function abort() 
{
  assertCanCommunicateWithServer $2
  turtle
  response=$(curl --connect-timeout $CURL_CONNECT_TIMEOUT --max-time $CURL_MAX_TIMEOUT -X POST --header "Content-Type: application/json" --header "Accept: application/json" "${2}/api/workflows/v1/${1}/abort")
  local r=$?
  echo $response  
  return $r
}

# List all jobs submitted in ${CROMWELL_SUBMISSIONS_FILE}
function list()
{
  local doColor=false
  local doUpdate=false

  # Handle Arguments:
  local OPTIND
  while getopts "cu" opt ; do
    case ${opt} in 
      c)
        doColor=true
        ;;
      u)
        doUpdate=true
        ;;
      *)
        invalidSubCommand list flag ${OPTARG}
        ;; 
    esac 
  done
  shift $((OPTIND-1)) 

  # Display the logo:
  turtle

  # If we need to update the data, do so here:
  if $doUpdate ; then
    error "Updating cached status list..."
    local tmpFile srv id runSt 

    # Use tput to save the cursor position:
    tput sc

    # The turtle starts 7 rows above where the cursor is at this point,
    # so if we need to move it, we have to move the cursor there before
    # we output anything:
    tput cuu 7

    # Reserve the name for a file whose non-existence will cause 
    # the following loop to exit.
    # This is good because all temp files made with `makeTemp` 
    # are automatically removed at exit, so this sub-process
    # will always die gracefully.
    local flagFile=$(makeTemp)
    echo 'TEST' > ${flagFile}

    # Kick off a turtle spinner...
    _turtleSpinner ${flagFile}

    # Make a copy of our file because we'll be modifying it:
    tmpFile=$( makeTemp )
    cp ${CROMWELL_SUBMISSIONS_FILE} ${tmpFile}

    while read line ; do 
      id=$( echo "$line" | awk '{print $3}' )
      # Only update if the ID field is, in fact, a valid ID:
      if [[ $(matchWorkflowId ${id}) -eq 0 ]] ; then
        srv=$( echo "$line" | awk '{print $2}' )
        runSt=$( echo "$line" | awk '{print $5}' )

        # get the status of the run if it has not completed:
        if [[ "${runSt}" != "Failed" ]] && [[ "${runSt}" != "Aborted" ]] && [[ "${runSt}" != "Succeeded" ]] ; then
          status ${id} ${srv} &> /dev/null
        fi
      fi
    done < ${tmpFile}

    # Remove the flag file to kill the spinner and wait for it to finish:
    rm ${flagFile}
    wait

    # Restore cursor position:
    tput rc
  fi

  # If we have to colorize the output, we do so:
  if $doColor ; then

    local tmpFile=$( makeTemp )  
    cat ${CROMWELL_SUBMISSIONS_FILE} | column -t > ${tmpFile}

    while read line ; do 

      # Check for header line:
      echo "${line}" | grep -q '^DATE' 
      r=$?
      [ $r -eq 0 ] && echo -e "${COLOR_UNDERLINED}${line}${COLOR_NORM}" && continue

      # Check for failed jobs and color those lines:
      echo "${line}" | grep -q 'Failed' 
      r=$?
      [ $r -eq 0 ] && echo -e "${COLOR_FAILED}${line}${COLOR_NORM}" && continue

      # Check for successful jobs and color those lines:
      echo "${line}" | grep -q 'Succeeded' 
      r=$?
      [ $r -eq 0 ] && echo -e "${COLOR_SUCCEEDED}${line}${COLOR_NORM}" && continue

      # Check for running jobs and color those lines:
      echo "${line}" | grep -q 'Running' 
      r=$?
      [ $r -eq 0 ] && echo -e "${COLOR_RUNNING}${line}${COLOR_NORM}" && continue

      echo "${line}"  
    done < ${tmpFile} 
  else  
    cat ${CROMWELL_SUBMISSIONS_FILE} | column -t
  fi
  return $?
}

function cleanup() 
{
  local st="" 

  # Handle Arguments:
  local OPTIND
  while getopts "s:" opt ; do
    case ${opt} in 
      s)
        local isGood=false
        for okVal in ${TERMINAL_STATES} ; do 
          if [[ "${OPTARG}" == "${okVal}" ]] ; then 
            isGood=true
          fi
        done 

        if ! $isGood ; then
          invalidSubCommand cleanup STATUS ${OPTARG}
        fi

        st=${OPTARG}
        ;;
      *)
        invalidSubCommand cleanup flag ${OPTARG}
        ;; 
    esac 
  done
  shift $((OPTIND-1)) 

  # Make sure we don't have any extra arguments here:
  if [[ $# -ne 0 ]] ; then
    invalidSubCommand cleanup flag "${@}"
  fi  

  # Get all states if we don't specify one:  
  if [[ ${#st} -eq 0 ]] ; then   
    st=${TERMINAL_STATES}
  fi

  # Display the logo:
  turtle

  echo  
  echo "State(s) selected: ${st}"
  echo

  getAnswerFromUser "Are you sure you want to CLEAR ALL RECORDS from these states: ${st}" 'Yes No' answer
  
  if [[ $( echo "${answer}" | tr A-Z a-z ) == "yes" ]] ; then
    echo "User answered 'Yes'."
    echo

    #Backup the file:
    echo "Creating backup of records file:"
    cp -v ${CROMWELL_SUBMISSIONS_FILE} ${CROMWELL_SUBMISSIONS_FILE}.$(date +%Y%m%dT%H%M%S ).bak
    echo

    # Remove the lines:
    local tmpFile=$( makeTemp )
    for s in ${st} ; do
      echo -n "Removing records with state: ${s}"

      # Go through each line and filter by status, which is the last column:
      while read line ; do 
        lineStatus=$( echo $line | awk '{print $NF}' )
        if [[ "${lineStatus}" != "${s}" ]] ; then
          echo $line
        fi  
      done < ${CROMWELL_SUBMISSIONS_FILE} > ${tmpFile}
    
      # Copy the temp file to the final file location for the next 
      # iteration or for when we're done
      cp ${tmpFile} ${CROMWELL_SUBMISSIONS_FILE}

      echo -e '\t\tDONE!'
    done 

    echo 'Your cromshell logs are now clean.'
    return 0
  else
    echo "Not clearing logs.  User aborted..."
    return 1
  fi
}

function assertDialogExists()
{
  which dialog &>/dev/null   
  r=$?
  [ $r -ne 0 ] && error "dialog does not exist.  Must install \`dialog\`: (yum install dialog / brew install dialog / apt-get dialog)." && exit 8
}

function assertGsUtilExists()
{
  which gsutil &>/dev/null   
  r=$?
  [ $r -ne 0 ] && error "gsutil does not exist.  Must install google cloud utilities." && exit 8
}

# List the output files for a given run.
# Will list all output files that are not:
#     *.log
#     rc
#     stdout
#     stderr
#     script
#     output
function list-outputs() 
{
  assertCanCommunicateWithServer $2
  turtle
  local id=$1
  local cromwellServer=$2

  local remoteFolder=$( metadata ${id} ${cromwellServer} | grep "\"callRoot\":" | head -n1 | awk '{print $2}' | sed "s#\"\\(.*${id}\\).*#\\1#g" )

  local localServerFolder="${CROMSHELL_CONFIG_DIR}/$( echo "${cromwellServer}" | sed -e 's#ht.*://##g' )/${id}"

  assertGsUtilExists

  error "Output files from job ${cromwellServer}:${id} : "

  # The -n flag just lists what would have happened if the copy occurred
  # we use this to our advantage to list the files we want:
  gsutil rsync -rP -n -x '.*\.[Ll][oO][Gg]$|.*rc|.*stdout|.*stderr|.*script|.*output' ${remoteFolder}/ ${localServerFolder}/. 2>&1 | \
    grep 'Would copy' | \
    sed 's#Would copy \(.*\) to .*#\1#g'

  return 0
}


# Get the root log folder from the cloud and put it in the metadata folder for this run
function fetch-logs() 
{
  assertCanCommunicateWithServer $2
  turtle
  local id=$1
  local cromwellServer=$2

  local remoteFolder=$( metadata ${id} ${cromwellServer} | grep "\"callRoot\":" | head -n1 | awk '{print $2}' | sed "s#\"\\(.*${id}\\).*#\\1#g" )

  local localServerFolder="${CROMSHELL_CONFIG_DIR}/$( echo "${cromwellServer}" | sed -e 's#ht.*://##g' )/${id}"

  assertGsUtilExists

  mkdir -p ${localServerFolder}

  error "Retrieving logs from ${remoteFolder}"
  error "Copying into ${localServerFolder}"

  # Do the copy, filtering for log files only:
  # Specifically, it will copy all files:
  #     - ending in .log (case insensitive)
  #     - Named rc
  #     - Named stdout
  #     - Named stderr
  #     - Named script
  gsutil rsync -rP -x '^(?!.*\.[Ll][oO][Gg]$|.*rc|.*stdout|.*stderr|.*script).*' ${remoteFolder}/ ${localServerFolder}/.

  error "Files copied to local folder: ${localServerFolder}"

  return 0
}

# Get the root log folder from the cloud and put it in the metadata folder for this run
# Includes all logs and output files
function fetch-all() 
{
  assertCanCommunicateWithServer $2
  turtle
  local id=$1
  local cromwellServer=$2

  local remoteFolder=$( metadata ${id} ${cromwellServer} | grep "\"callRoot\":" | head -n1 | awk '{print $2}' | sed "s#\"\\(.*${id}\\).*#\\1#g" )

  local localServerFolder="${CROMSHELL_CONFIG_DIR}/$( echo "${cromwellServer}" | sed -e 's#ht.*://##g' )/${id}"

  assertGsUtilExists

  mkdir -p ${localServerFolder}

  error "Retrieving logs from ${remoteFolder}"
  error "Copying into ${localServerFolder}"
  gsutil rsync -rP ${remoteFolder}/ ${localServerFolder}/.

  error "Files copied to local folder: ${localServerFolder}"

  return 0
}

function assertValidEmail() 
{
  # Make sure the user gave us a good email address:
  echo "$1" | grep -q  '@' 
  r=$?
  if [ $r -ne 0 ] ; then
    error "Error: invalid email address: $1"
    error ""
    simpleUsage notify
    echo "Try \`$SCRIPTNAME -h' for more information."
    exit 5
  fi
}

# Spin off a task in the background that will check periodically if the workflow ID is 
# complete.
# When it is, send an email to the email address specified.
function notify()
{
  if [[ $# -lt 1 ]] ; then
    error "Error: You must specify an email address to notify."
    error ""
    simpleUsage notify
    echo "Try \`$SCRIPTNAME -h' for more information."
    exit 5
  fi

  # Is the next argument our workflow ID?
  # Workflow IDs are standard UUIDs or negative numbers:
  if [[ $(matchWorkflowId ${1}) -eq 0 ]] || [[ ${1} =~ ^-[[:digit:]]+$ ]] ; then
    # Get the workflow ID:
    populateWorkflowIdAndServerUrl $1
    shift
  fi

  # Get the host on which this will be run:
  hostServer="${HOSTNAME}"

  if [[ ! "${1}" =~ ^.*@.*  ]] ; then
    hostServer=$1
    shift
  fi

  # Make sure the user gave us a good email address:
  assertValidEmail $1
  email=$1
  shift

  # Make sure our workflow ID is populated:
  populateWorkflowIdAndServerUrl ${WORKFLOW_ID}

  # Make sure we have a workflow server defined.
  # This is important if we're on a remote server for the notification
  # daemon.
  if [[ ${#WORKFLOW_SERVER_URL} -eq 0 ]] ; then
    if [[ ${#1} -ne 0 ]] ; then
      WORKFLOW_SERVER_URL="${1}"
      shift
    else
      error 'Error: Could not determine the cromwell server url!'
      exit 2
    fi
  fi

  # Do some wizardry here to spin off the notification daemon on a remote server:
  if [[ "${hostServer}" != "${HOSTNAME}" ]] ; then
    turtle
    error "Creating notification daemon on host ${hostServer} ..."

    # Send the script to the server:
		local tmpOut=$( makeTemp )
    scp ${SCRIPTDIR}/${SCRIPTNAME} ${hostServer}:~/. &> ${tmpOut}
    [[ $? -ne 0 ]] && error "ERROR: Could not copy cromshell to server ${hostServer}" && error "$(cat ${tmpOut})" && exit 7

    # Spin off notification process on the server:
    results=$( ssh ${hostServer} "~/${SCRIPTNAME} notify ${WORKFLOW_ID} ${email} ${WORKFLOW_SERVER_URL}" )
    [[ $? -ne 0 ]] && error "ERROR: Could not start notification daemon on ${hostServer}" && exit 8

    # Let the user know we've done our job:
    echo "Spun off thread on ${hostServer} - PID = $( echo "${results}" | grep "Spun off thread on PID" | sed 's#Spun off thread on PID ##g' )"
  else
    error "Spinning off notification to ${email} thread for"
    error "    workflow:             ${WORKFLOW_ID}" 
    error "    from Cromwell server: ${WORKFLOW_SERVER_URL}"
    error "..."
    nohup bash -c "source ${SCRIPTDIR}/${SCRIPTNAME}; _notifyHelper ${WORKFLOW_ID} ${email} ${WORKFLOW_SERVER_URL}" &>/dev/null &
    echo "Spun off thread on PID $!"
  fi
}

# Helper function for the notify command that actually notifies the user:
function _notifyHelper()
{
  local id=$1
  local email=$2
  local cromwellServer=$3

  assertCanCommunicateWithServer ${cromwellServer}
  local separator="=================================================="

  while true ; do 
    completionStatus=$( status $id ${cromwellServer} 2>/dev/null | grep "status" | sed -e 's#.*status":##g' | tr -d ' ",' )
    echo $completionStatus | grep -qE "Succeeded|Failed|Aborted" 
    r=${PIPESTATUS[1]}
    if [ $r -eq 0 ] ; then
      workflowFile=$( grep '$id' ${CROMWELL_SUBMISSIONS_FILE} | head -n1 | awk '{print $4}' ) 
      metaData=$( metadata $id ${cromwellServer} )
      echo -e "CROMWELL Task ${completionStatus}:\n\n${id}\n\non\n\n${cromwellServer}\n\n${separator}\n\nStatus:\n$(cat ${statusFile})\n\n${separator}\nMetadata:\n${metaData}\n\n${separator}\nSent by $( whoami )@$( hostname ) on $( date ) \n\n\n" | mail -n -s "Cromwell Task ${completionStatus} [${cromwellServer}] ${workflowFile}" ${email} 
      break
    fi
    # wait for 10 seconds:
    sleep 10
  done 
}

function runSubCommandOnWorkflowId()
{
  # Get the workflow ID:
  populateWorkflowIdAndServerUrl $1
  error "Using workflow-id == $WORKFLOW_ID"
  error "Using workflow server URL == $WORKFLOW_SERVER_URL"
  ${SUB_COMMAND} ${WORKFLOW_ID} ${WORKFLOW_SERVER_URL} 
  r=$?
  echo
  return $r
}

function dialogAskUserIfUrlIsOK()
{
  local BACKTITLE=$1
  local url=$2
  local menuFile=$3

  # Dialog to validate that the user has entered the correct thing:
  dialog --clear --backtitle "${BACKTITLE}" --title "Cromwell URL Verification" \
    --yesno "Oh my!  It looks like your server is: \n\
\n\
${url}\n\
\n\
Is this correct?" 10 75 2>${menuFile}
  local r=$?

  return $r 
}

function checkForUserQuitDialogEarly()
{
  # If the user gets cold feet, let them go softly into the night.
  if [[ $1 -ne 0 ]] ; then
    clear 
    error "   User aborted setup."
    error ""
    turtle
    error ""
    error "  Please come back soon!"
    exit 6
  fi
}

function setupCromshellDialog()
{
  assertDialogExists

  local menuFile=$( makeTemp )
  local splashTextFile=$( makeTemp )

  # Menu variables:
  local TITLE="Welcome to Cromshell!"
  local BACKTITLE="Cromshell First Time Setup"
  local URL_QUESTION_TEXT="What is the URL of the Cromwell server you use to execute your jobs?"
  local r

  # Set up our logo:
  turtle &> ${menuFile} 
  sed 's#^#     #g' ${menuFile} > ${splashTextFile}
  
  # Splash screen:
  dialog --beep --exit-label "Continue" --backtitle "${BACKTITLE}" --title "${TITLE}" --textbox ${splashTextFile} 12 43
  r=$?

  # URL input dialog:
  dialog --clear --backtitle "${BACKTITLE}" --title "${TITLE}" \
    --inputbox "This seems to be your first time running me, Cromshell! \n\
(Please excuse me if it isn't!  I seem to have forgotten some of your information!)\n\
\n\
I am your personal, personable, and wonderful extension of the Cromwell API right here on the command-line.\n\
\n\
There are a few small questions I need you to answer before I can make your cromwell dreams come true.\n\
\n\
${URL_QUESTION_TEXT}" 20 75 2> ${menuFile} 
  r=$?

  checkForUserQuitDialogEarly $r

  local hasGoodUrl=false

  # Get the URL from the menu file:
  url=$( cat ${menuFile} | sed -e 's#^[ \t]*##g' -e 's#[ \t]*$##g' )

  # Dialog to validate that the user has entered the correct thing:
  dialogAskUserIfUrlIsOK "${BACKTITLE}" "${url}" "${menuFile}"
  r=$?

  # See if the user is OK with their URL:
  if [[ $r -eq 0 ]] ; then 
    hasGoodUrl=true
  fi

  # Until we have a good URL, we need to keep asking the user.
  while ! $hasGoodUrl ; do 

    # URL input dialog:
    dialog --clear \
      --backtitle "${BACKTITLE}" \
      --title "Cromwell URL Verification" \
      --inputbox "Oh, I'm terribly sorry for that.  Let's try again! \n\
\n\
${URL_QUESTION_TEXT}" 10 75 2> ${menuFile} 
    r=$?

    checkForUserQuitDialogEarly $r

    # Get the URL from the menu file:
    url=$( cat ${menuFile} | sed -e 's#^[ \t]*##g' -e 's#[ \t]*$##g' )

    # Dialog to validate that the user has entered the correct thing:
    dialogAskUserIfUrlIsOK "${BACKTITLE}" "${url}" "${menuFile}"
    r=$?

    # See if the user is OK with their URL:
    if [[ $r -eq 0 ]] ; then 
      hasGoodUrl=true
    fi
  done
  
  # Put the URL in the file:
  echo "${url}" > ${CROMWELL_SERVER_FILE}

  # Thank the user for their input and let them know 
  # they are ready to run:
  echo -e "" > ${splashTextFile}
  echo -e "OK.  I have set your Cromwell server to be:" >> ${splashTextFile}
  echo -e "" >> ${splashTextFile}
  echo -e "${url}" >> ${splashTextFile}
  echo -e "" >> ${splashTextFile}
  echo -e "You should be ALL SET!" >> ${splashTextFile}
  echo -e "Please run me again to start the executions." >> ${splashTextFile}

  dialog --clear --beep --exit-label "Continue" --backtitle "${BACKTITLE}" --title "${TITLE}" --textbox ${splashTextFile} 13 75
  r=$?

  clear

  return 0
}

function setupCromshell() 
{
  echo "--------------------------------------------------"
  echo -e "   __        __   _                          "
  echo -e "   \\ \\      / /__| | ___ ___  _ __ ___   ___ "
  echo -e "    \\ \\ /\\ / / _ \\ |/ __/ _ \\| '_ \` _ \\ / _ \\"
  echo -e "     \\ V  V /  __/ | (_| (_) | | | | | |  __/"
  echo -e "      \\_/\\_/ \\___|_|\\___\\___/|_| |_| |_|\\___|"
  echo -e "                                             "
  echo -e "                    _        "
  echo -e "                   | |_ ___  "
  echo -e "                   | __/ _ \\ "
  echo -e "                   | || (_) |"
  echo -e "                    \\__\\___/ "
  echo -e "                             "
  echo -e "   ____                         _          _ _ "
  echo -e "  / ___|_ __ ___  _ __ ___  ___| |__   ___| | |"
  echo -e " | |   | '__/ _ \\| '_ \` _ \\/ __| '_ \\ / _ \\ | |"
  echo -e " | |___| | | (_) | | | | | \__ \ | | |  __/ | |"
  echo -e "  \\____|_|  \\___/|_| |_| |_|___/_| |_|\\___|_|_|"
  echo -e "                                               "
  
  echo
  turtle
  echo

  echo "--------------------------------------------------"

  echo
  echo 'This seems to be your first time running me, Cromshell!'
  echo "(Please excuse me if it isn't!  I seem to have forgotten"
  echo 'some of your information!)'
  echo ''
  echo "I am your personal, personable, and wonderful extension of "
  echo "the Cromwell API right here on the command-line."
  echo ""
  echo "There are a few small questions I need you to answer before"
  echo "I can make your cromwell dreams come true."
  echo
  echo "--------------------------------------------------"
  echo

  # Get the server they need to connect to:
  local choseServer=false
  while ! $choseServer ; do
    echo "What is the URL of the Cromwell server you use to execute"
    echo "your jobs?"

    # Read in the URL:
    read -p "Cromwell URL, please: " url
    echo

    # Force the user to acknowledge their input:
    getAnswerFromUser "Oh my!  It looks like your server is: ${url}$(echo $'\nIs this correct')" 'Yes No' answer

    if [[ $( echo "${answer}" | tr A-Z a-z ) == "yes" ]] ; then
      choseServer=true
    else
      echo "Oh, I'm terribly sorry for that.  Let's try again!"
      echo
      echo
    fi
  done

  echo
  echo "OK.  I'm now setting your Cromwell server to be: ${url}"
  echo -n "Don't worry - this won't hurt a bit..."
  
  # Put the URL in the file:
  echo "${url}" > ${CROMWELL_SERVER_FILE}
  echo "DONE"

  echo
  echo "OK, you should be ALL SET!"
  echo "Please run me again to start the executions."
}

################################################################################

# Do all the interactive stuff if the script is called by a user.
# The bash equivalent to `if __name__ == '__main__':
if ${ISINTERACTIVESHELL} ; then

  # Check the remaining arguments for help and display it if we have to: 
  checkForComplexHelpArgument $@

  # Get flags:
  while getopts ":t:" opt ; do
    case ${opt} in 
      t)
        # Ensure that the value given to -t is numerical:
        echo ${OPTARG} | grep -q -E '^[0-9]+$' 
        rv=$?
        [ $rv -ne 0 ] && invalidSubCommand '' "flag value (${OPTARG})" "-${opt} argument must be an integer."

        # Set our timeouts:
        CURL_CONNECT_TIMEOUT=${OPTARG}
        let CURL_MAX_TIMEOUT=2*${CURL_CONNECT_TIMEOUT}
        ;;
      :)
        invalidSubCommand '' 'flag value' "-${opt} requires an argument."
        ;;
      *)
        invalidSubCommand '' flag ${OPTARG}
        ;; 
    esac 
  done
  shift $((OPTIND-1)) 

  # Get our sub-command:
  SUB_COMMAND=${1}
  shift
  
  # Check if we need to set this up.
  # Note: because of how `notify` works, we can't require the setup for the notify action.
  if ${CROMWELL_NEEDS_SETUP} && [[ "${SUB_COMMAND}" != "notify" ]] ; then
    # We need to setup this install.
    # Now let's set it up:
    which dialog &> /dev/null
    r=$?
     if [[ $r -eq 0 ]] ; then
      setupCromshellDialog
    else
      setupCromshell
    fi
    exit 0
  fi

  # Validate our sub-command:
  case ${SUB_COMMAND} in
    cleanup|submit|status|logs|execution-status-count|metadata|slim-metadata|timing|abort|notify|list|fetch-all|fetch-logs|list-outputs)
      # This is a good sub-command, so we do not need to do anything. 
      ;;
    *)
      if [[ ${#SUB_COMMAND} -eq 0 ]]; then
        simpleUsage 
        exit 0
      else   
        invalidSubCommand '' "sub-command" "${SUB_COMMAND}"
      fi
      ;; 
  esac

  # Now that we have checked for help arguments, we can
  # make sure we have all the required commands:
  checkPrerequisites $PREREQUISITES 
  r=$?
  [[ $r -ne 0 ]] && exit 6

  # Handle specific sub-command args and and call our sub-command:
  echo "Sub-Command: ${SUB_COMMAND}"
  case ${SUB_COMMAND} in 
    cleanup|submit|list|notify)
      ${SUB_COMMAND} $@
      rv=$?
      ;;
    *)
      # All other tools just take a list of workflow IDs:
      handleArgs_workflow_ids $@

      for WORKFLOW_ID in ${WORKFLOW_ID_LIST} ; do 
        runSubCommandOnWorkflowId ${WORKFLOW_ID}
        rv=$?
      done
      ;;
  esac

  exit $rv
fi