Various...

Add cookie-based authentication while still supporting basic auth headers
Fix quicktunnel startup logic
Replace lsof with fuser for killing processes
Check network volumes for compatibility with permissions changes
update docs to reflect recent changes

build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh

@@ -83,11 +83,11 @@ $APT_INSTALL \
  websockets
 
 # Get caddy server
-mkdir -p /opt/caddy/bin
-wget -c -O caddy.tar.gz https://github.com/caddyserver/caddy/releases/download/v2.7.5/caddy_2.7.5_linux_amd64.tar.gz 
-tar -xf caddy.tar.gz -C /opt/caddy
-rm caddy.tar.gz
-mv /opt/caddy/caddy /opt/caddy/bin
+#mkdir -p /opt/caddy/bin
+#wget -c -O caddy.tar.gz https://github.com/caddyserver/caddy/releases/download/v2.7.5/caddy_2.7.5_linux_amd64.tar.gz 
+#tar -xf caddy.tar.gz -C /opt/caddy
+#rm caddy.tar.gz
+#mv /opt/caddy/caddy /opt/caddy/bin
 
 # Get Cloudflare daemon
 wget -c -O cloudflared.deb https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb

build/COPY_ROOT/opt/ai-dock/bin/cfnt-url.sh

@@ -7,10 +7,12 @@ function cleanup() {
 }
 
 unset -v port
-metrics=""
-while getopts p: flag
+unset -v location
+
+while getopts l:p: flag
 do
  case "${flag}" in
+ l) location="${OPTARG}";;
  p) port="${OPTARG}";;
  esac
 done
@@ -20,16 +22,16 @@ if [[ -z $port ]]; then
  exit 1
 fi
 
-listen_port=$(cat /run/http_ports/${port} | jq -r '.listen_port' 2>/dev/null)
-ingress_json=$(curl -s http://localhost:2999/config | jq -r .config.ingress 2>/dev/null)
-ingress_count=$(printf "%s" "$ingress_json" | jq length 2>/dev/null)
+listen_port=2999
+ingress_json="$(curl -s http://localhost:${listen_port}/config | jq -r .config.ingress 2>/dev/null)"
+ingress_count="$(printf "%s" "$ingress_json" | jq length 2>/dev/null)"
 
 for ((i=0;i<ingress_count;i++)); do
- ingress=$(printf "%s" "$ingress_json" | jq -r ".[${i}]" 2>/dev/null)
- service_port=$(printf "%s" "$ingress" | jq -r .service | cut -d ":" -f 3 2>/dev/null)
+ ingress="$(printf "%s" "$ingress_json" | jq -r ".[${i}]" 2>/dev/null)"
+ service_port="$(printf "%s" "$ingress" | jq -r .service | cut -d ":" -f 3 2>/dev/null)"
 
- if [[ $service_port = $port ]]; then
- printf "https://%s\n" $(echo "$ingress" | jq -r .hostname 2>/dev/null)
+ if [[ $service_port == "$port" ]]; then
+ printf "https://%s%s\n" "$(printf "%s" "$ingress" | jq -r .hostname 2>/dev/null)" "$location"
  exit 0
  fi
 done

build/COPY_ROOT/opt/ai-dock/bin/cfqt-url.sh

@@ -7,10 +7,12 @@ function cleanup() {
 }
 
 unset -v port
-metrics=""
-while getopts p: flag
+unset -v location
+
+while getopts l:p: flag
 do
  case "${flag}" in
+ l) location="${OPTARG}";;
  p) port="${OPTARG}";;
  esac
 done
@@ -20,10 +22,13 @@ if [[ -z $port ]]; then
  exit 1
 fi
 
+mport=$(jq -r .metrics_port /run/http_ports/$port 2>/dev/null)
+if [[ -n $mport ]]; then
+ cf_host=$(curl -s http://localhost:${mport}/quicktunnel | jq -r .hostname 2>/dev/null)
+fi
 
-
-if mport=$(jq -r .metrics_port /run/http_ports/$port 2>/dev/null) && cf_host=$(curl -s http://localhost:${mport}/quicktunnel | jq -r .hostname 2>/dev/null); then
- printf "https://%s\n" $cf_host
+if [[ -n $mport && -n $cf_host ]]; then
+ printf "https://%s%s\n" "$cf_host" "$location"
  exit 0
 else
  printf "No cloudflare quicktunnel running for localhost:%s\n" $port

build/COPY_ROOT/opt/ai-dock/bin/direct-url

@@ -0,0 +1 @@
+direct-url.sh

build/COPY_ROOT/opt/ai-dock/bin/direct-url.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+trap cleanup EXIT
+
+function cleanup() {
+ kill $(jobs -p) >/dev/null 2>&1
+}
+
+unset -v port
+unset -v url
+
+metrics=""
+while getopts l:p: flag
+do
+ case "${flag}" in
+ l) location="${OPTARG}";;
+ p) port="${OPTARG}";;
+ esac
+done
+
+if [[ -z $port ]]; then
+ printf "port (-p) is required\n"
+ exit 1
+fi
+
+function get_url {
+ # Vast.ai
+ if [[ $DIRECT_ADDRESS == "auto#vast-ai" ]]; then
+ declare -n vast_mapped_port=VAST_TCP_PORT_${port}
+ if [[ -n $vast_mapped_port && -n $PUBLIC_IPADDR ]]; then
+ url="http://${PUBLIC_IPADDR}:${vast_mapped_port}"
+ fi
+ # Runpod.io
+ elif [[ $DIRECT_ADDRESS == "auto#runpod-io" ]]; then
+ declare -n runpod_mapped_port=RUNPOD_TCP_PORT_${port}
+ if [[ -n $runpod_mapped_port && -n $RUNPOD_PUBLIC_IP ]]; then
+ url="http://${RUNPOD_PUBLIC_IP}:${runpod_mapped_port}"
+ elif [[ -n $RUNPOD_POD_ID ]]; then
+ url="https://${RUNPOD_POD_ID}-${port}.proxy.runpod.net"
+ fi
+ # Other cloud / local
+ else
+ url="http://${DIRECT_ADDRESS}:${port}"
+ fi
+
+ if [[ -n $url ]]; then
+ printf "%s%s\n" "$url" "$location"
+ exit 0
+ else
+ printf "Could not create URL\n"
+ exit 1
+ fi
+}
+
+get_url

build/COPY_ROOT/opt/ai-dock/bin/fix-permissions.sh

@@ -26,15 +26,19 @@ function fix_container() {
 }
 
 function fix_workspace() {
- printf "Fixing workspace permissions...\n"
- chown -R ${WORKSPACE_UID}.${WORKSPACE_GID} "${WORKSPACE}"
- chmod -R g+s ${WORKSPACE}
- setfacl -R -d -m u:"${WORKSPACE_UID}":rwx "${WORKSPACE}"
- setfacl -R -d -m m:rwx "${WORKSPACE}"
- chmod o-rw ${WORKSPACE}/home/user
- if [[ -e ${WORKSPACE}/home/user/.ssh/authorized_keys ]]; then
- chmod 700 ${WORKSPACE}/home/user/.ssh
- chmod 600 ${WORKSPACE}/home/user/.ssh/authorized_keys
+ if [[ $WORKSPACE_PERMISSIONS == "true" ]]; then
+ printf "Fixing workspace permissions...\n"
+ chown -R ${WORKSPACE_UID}.${WORKSPACE_GID} "${WORKSPACE}"
+ chmod -R g+s ${WORKSPACE}
+ setfacl -R -d -m u:"${WORKSPACE_UID}":rwx "${WORKSPACE}"
+ setfacl -R -d -m m:rwx "${WORKSPACE}"
+ chmod o-rw ${WORKSPACE}/home/user
+ if [[ -e ${WORKSPACE}/home/user/.ssh/authorized_keys ]]; then
+ chmod 700 ${WORKSPACE}/home/user/.ssh
+ chmod 600 ${WORKSPACE}/home/user/.ssh/authorized_keys
+ fi
+ else
+ printf "No permissions changed (non-standard fs)\n"
  fi
 }
 

build/COPY_ROOT/opt/ai-dock/bin/init.sh

@@ -122,19 +122,26 @@ function init_set_ssh_keys() {
 }
 
 init_set_web_credentials() {
+ export SERVICEPORTAL_LOGIN=$(direct-url.sh -p "${SERVICEPORTAL_PORT_HOST:-1111}" -l "/login")
+ export SERVICEPORTAL_HOME=$(direct-url.sh -p "${SERVICEPORTAL_PORT_HOST:-1111}")
+
  if [[ -z $WEB_USER ]]; then
- WEB_USER=user
+ export WEB_USER=user
+ fi
+
+ if [[ -z $WEB_PASSWORD ]]; then
+ export WEB_PASSWORD="$(openssl rand -base64 12)"
  fi
 
- if [[ -z $WEB_PASSWORD && -z $WEB_PASSWORD_HASH ]]; then
- WEB_PASSWORD=password
- elif [[ -z $WEB_PASSWORD ]]; then
- WEB_PASSWORD="********"
+ export WEB_PASSWORD_B64="$(printf "%s:%s" "$WEB_USER" "$WEB_PASSWORD" | base64)"
+
+ if [[ -z $WEB_TOKEN ]]; then
+ # Not the same as password (probably!)
+ export WEB_TOKEN="$(openssl rand -base64 32)"
  fi
 
- if [[ $WEB_PASSWORD != "********" ]]; then
- WEB_PASSWORD_HASH=$(hash-password.sh -p $WEB_PASSWORD -r 15)
- export WEB_PASSWORD="********"
+ if [[ -n $DISPLAY && -z $COTURN_PASSWORD ]]; then
+ export COTURN_PASSWORD="auto_$(openssl rand -base64 8)"
  fi
 
  printf "%s %s" "$WEB_USER" "$WEB_PASSWORD_HASH" > /opt/caddy/etc/basicauth
@@ -156,10 +163,13 @@ function init_count_gpus() {
 }
 
 function init_count_quicktunnels() {
- if [[ ! ${CF_QUICK_TUNNELS,,} = "true" ]]; then
+ if [[ ${CF_QUICK_TUNNELS,,} == "false" ]]; then
  export CF_QUICK_TUNNELS_COUNT=0
  else
- export CF_QUICK_TUNNELS_COUNT=$(grep -l "METRICS_PORT" /opt/ai-dock/bin/supervisor-*.sh | wc -l)
+ export CF_QUICK_TUNNELS_COUNT=$(grep -l "QUICKTUNNELS=true" /opt/ai-dock/bin/supervisor-*.sh | wc -l)
+ if [[ -z $TUNNEL_TRANSPORT_PROTOCOL ]]; then
+ export TUNNEL_TRANSPORT_PROTOCOL=http2
+ fi
  fi
 }
 
@@ -199,6 +209,17 @@ function init_set_workspace() {
  touch "${no_mount_warning_file}"
  printf "%b" "${no_mount_warning}" > "${no_mount_warning_file}"
  fi
+ # Ensure we have a proper linux filesystem so we don't run into errors on sync
+ if [[ $WORKSPACE_MOUNTED == "true" ]]; then
+ test_file=${WORKSPACE}/.ai-dock-permissions-test
+ touch $test_file
+ if chown ${WORKSPACE_UID}.${WORKSPACE_GID} $test_file; then
+ export WORKSPACE_PERMISSIONS=true
+ else 
+ export WORKSPACE_PERMISSIONS=false
+ fi
+ rm $test_file
+ fi
 }
 
 # This is a convenience for X11 containers and bind mounts - No additional security implied.
@@ -208,7 +229,7 @@ function init_create_user() {
  mkdir -p ${home_dir}
  groupadd -g $WORKSPACE_GID $USER_NAME
  useradd -ms /bin/bash $USER_NAME -d $home_dir -u $WORKSPACE_UID -g $WORKSPACE_GID
- printf "user:%s" "$USER_PASSWORD" | chpasswd
+ printf "user:%s" "${USER_PASSWORD}" | chpasswd
  usermod -a -G $USER_GROUPS $USER_NAME
  # May not exist - todo check device ownership
  usermod -a -G render $USER_NAME
@@ -252,7 +273,11 @@ function init_sync_mamba_envs() {
  mkdir -p ${WORKSPACE}/environments
  printf "Moving mamba environments to %s...\n" "${WORKSPACE}"
  while sleep 10; do printf "Waiting for workspace mamba sync...\n"; done &
- rsync -auSHh --stats /opt/micromamba/ "${ws_mamba_target}"
+ if [[ $WORKSPACE_PERMISSIONS == "true" ]]; then
+ rsync -auSHh --stats /opt/micromamba/ "${ws_mamba_target}"
+ else
+ rsync -uSHh --stats /opt/micromamba/ "${ws_mamba_target}"
+ fi
  kill $!
  wait $! 2>/dev/null
  printf "Moved mamba environments to %s\n" "${WORKSPACE}"

build/COPY_ROOT/opt/ai-dock/bin/set-web-credentials.sh

@@ -5,11 +5,14 @@ if [[ -z $2 ]]; then
  exit 1
 fi
 
-WEB_USER=$1
-WEB_PASSWORD_HASH=$(hash-password.sh -p $2 -r 15)
-export WEB_PASSWORD="********"
+export WEB_USER=$1
+env-store WEB_USER
+export WEB_PASSWORD=$2
+env-store WEB_PASSWORD
+export WEB_PASSWORD_B64="$(printf "%s:%s" "$WEB_USER" "$WEB_PASSWORD" | base64)"
+env-store WEB_PASSWORD_B64
 
 printf "Setting credentials and restarting proxy server...\n"
-printf "%s %s" "$WEB_USER" "$WEB_PASSWORD_HASH" > /opt/caddy/etc/basicauth
-supervisorctl restart caddy
 
+supervisorctl restart serviceportal
+supervisorctl restart caddy

build/COPY_ROOT/opt/ai-dock/bin/supervisor-caddy.sh

@@ -31,10 +31,6 @@ function start() {
  template_file="/opt/caddy/share/service_config"
  fi
 
- if [[ ${WEB_ENABLE_AUTH,,} != 'false' && ${proxy_secure,,} != 'false' ]]; then
- template_file="${template_file}_auth"
- fi
-
  cp "${template_file}" /tmp/caddy
  sed -i "s/!PROXY_PORT/${proxy_port}/g" /tmp/caddy
  sed -i "s/!LISTEN_PORT/${listen_port}/g" /tmp/caddy

build/COPY_ROOT/opt/ai-dock/bin/supervisor-cloudflared.sh

@@ -5,7 +5,7 @@ trap cleanup EXIT
 METRICS_PORT=2999
 
 function cleanup() {
- kill $(lsof -t -i:${METRICS_PORT}) > /dev/null 2>&1 &
+ fuser -k -SIGTERM ${METRICS_PORT}/tcp > /dev/null 2>&1 &
  wait -n
 }
 
@@ -20,7 +20,7 @@ function start() {
 
  printf "Starting Cloudflare daemon...\n"
 
- kill -9 $(lsof -t -i:${METRICS_PORT}) > /dev/null 2>&1 &
+ fuser -k -SIGKILL ${METRICS_PORT}/tcp > /dev/null 2>&1 &
  wait -n
 
  cloudflared tunnel --metrics localhost:"${METRICS_PORT}" run --token "${CF_TUNNEL_TOKEN}"

build/COPY_ROOT/opt/ai-dock/bin/supervisor-quicktunnel.sh

@@ -3,7 +3,7 @@
 trap cleanup EXIT
 
 function cleanup() {
- kill $(lsof -t -i:${metrics_port}) > /dev/null 2>&1 &
+ fuser -k -SIGTERM ${metrics_port}/tcp > /dev/null 2>&1 &
  wait -n
 }
 
@@ -31,7 +31,7 @@ function start() {
  fi
 
  # Ensure the port is available (kill stale for restart)
- kill -9 $(lsof -t -i:${metrics_port}) > /dev/null 2>&1 &
+ fuser -k -SIGKILL ${metrics_port}/tcp > /dev/null 2>&1 &
  wait -n
 
  cloudflared tunnel ${metrics} ${tunnel}

build/COPY_ROOT/opt/ai-dock/bin/supervisor-serviceportal.sh

@@ -2,16 +2,16 @@
 
 trap cleanup EXIT
 
-LISTEN_PORT=${SERVICEPORTAL_PORT_LOCAL:-11111}
+LISTEN_PORT=11111
 METRICS_PORT=${SERVICEPORTAL_METRICS_PORT:-21111}
 PROXY_PORT=${SERVICEPORTAL_PORT_HOST:-1111}
-# Auth is true for defined paths - See /opt/caddy/share/service_config_11111_auth
-PROXY_SECURE=true
+QUICKTUNNELS=true
+
 SERVICE_NAME="Service Portal"
 
 function cleanup() {
  rm /run/http_ports/$PROXY_PORT > /dev/null 2>&1
- kill $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 &
+ fuser -k -SIGTERM ${LISTEN_PORT}/tcp > /dev/null 2>&1 &
  wait -n
 }
 
@@ -37,7 +37,7 @@ function start() {
 
  printf "Starting ${SERVICE_NAME}...\n"
 
- kill -9 $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 &
+ fuser -k -SIGKILL ${LISTEN_PORT}/tcp > /dev/null 2>&1 &
  wait -n
 
  /usr/bin/python3 /opt/ai-dock/fastapi/serviceportal/main.py \