diff --git a/build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh b/build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh index a182c5a..fac7484 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh @@ -83,11 +83,11 @@ $APT_INSTALL \ websockets # Get caddy server -mkdir -p /opt/caddy/bin -wget -c -O caddy.tar.gz https://github.com/caddyserver/caddy/releases/download/v2.7.5/caddy_2.7.5_linux_amd64.tar.gz -tar -xf caddy.tar.gz -C /opt/caddy -rm caddy.tar.gz -mv /opt/caddy/caddy /opt/caddy/bin +#mkdir -p /opt/caddy/bin +#wget -c -O caddy.tar.gz https://github.com/caddyserver/caddy/releases/download/v2.7.5/caddy_2.7.5_linux_amd64.tar.gz +#tar -xf caddy.tar.gz -C /opt/caddy +#rm caddy.tar.gz +#mv /opt/caddy/caddy /opt/caddy/bin # Get Cloudflare daemon wget -c -O cloudflared.deb https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb diff --git a/build/COPY_ROOT/opt/ai-dock/bin/cfnt-url.sh b/build/COPY_ROOT/opt/ai-dock/bin/cfnt-url.sh index 60adfbc..592a7f3 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/cfnt-url.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/cfnt-url.sh @@ -7,10 +7,12 @@ function cleanup() { } unset -v port -metrics="" -while getopts p: flag +unset -v location + +while getopts l:p: flag do case "${flag}" in + l) location="${OPTARG}";; p) port="${OPTARG}";; esac done @@ -20,16 +22,16 @@ if [[ -z $port ]]; then exit 1 fi -listen_port=$(cat /run/http_ports/${port} | jq -r '.listen_port' 2>/dev/null) -ingress_json=$(curl -s http://localhost:2999/config | jq -r .config.ingress 2>/dev/null) -ingress_count=$(printf "%s" "$ingress_json" | jq length 2>/dev/null) +listen_port=2999 +ingress_json="$(curl -s http://localhost:${listen_port}/config | jq -r .config.ingress 2>/dev/null)" +ingress_count="$(printf "%s" "$ingress_json" | jq length 2>/dev/null)" for ((i=0;i/dev/null) - service_port=$(printf "%s" "$ingress" | jq -r .service | cut -d ":" -f 3 2>/dev/null) + ingress="$(printf "%s" "$ingress_json" | jq -r ".[${i}]" 2>/dev/null)" + service_port="$(printf "%s" "$ingress" | jq -r .service | cut -d ":" -f 3 2>/dev/null)" - if [[ $service_port = $port ]]; then - printf "https://%s\n" $(echo "$ingress" | jq -r .hostname 2>/dev/null) + if [[ $service_port == "$port" ]]; then + printf "https://%s%s\n" "$(printf "%s" "$ingress" | jq -r .hostname 2>/dev/null)" "$location" exit 0 fi done diff --git a/build/COPY_ROOT/opt/ai-dock/bin/cfqt-url.sh b/build/COPY_ROOT/opt/ai-dock/bin/cfqt-url.sh index bd24382..b5c9818 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/cfqt-url.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/cfqt-url.sh @@ -7,10 +7,12 @@ function cleanup() { } unset -v port -metrics="" -while getopts p: flag +unset -v location + +while getopts l:p: flag do case "${flag}" in + l) location="${OPTARG}";; p) port="${OPTARG}";; esac done @@ -20,10 +22,13 @@ if [[ -z $port ]]; then exit 1 fi +mport=$(jq -r .metrics_port /run/http_ports/$port 2>/dev/null) +if [[ -n $mport ]]; then + cf_host=$(curl -s http://localhost:${mport}/quicktunnel | jq -r .hostname 2>/dev/null) +fi - -if mport=$(jq -r .metrics_port /run/http_ports/$port 2>/dev/null) && cf_host=$(curl -s http://localhost:${mport}/quicktunnel | jq -r .hostname 2>/dev/null); then - printf "https://%s\n" $cf_host +if [[ -n $mport && -n $cf_host ]]; then + printf "https://%s%s\n" "$cf_host" "$location" exit 0 else printf "No cloudflare quicktunnel running for localhost:%s\n" $port diff --git a/build/COPY_ROOT/opt/ai-dock/bin/direct-url b/build/COPY_ROOT/opt/ai-dock/bin/direct-url new file mode 120000 index 0000000..efddc8a --- /dev/null +++ b/build/COPY_ROOT/opt/ai-dock/bin/direct-url @@ -0,0 +1 @@ +direct-url.sh \ No newline at end of file diff --git a/build/COPY_ROOT/opt/ai-dock/bin/direct-url.sh b/build/COPY_ROOT/opt/ai-dock/bin/direct-url.sh new file mode 100755 index 0000000..9d49601 --- /dev/null +++ b/build/COPY_ROOT/opt/ai-dock/bin/direct-url.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +trap cleanup EXIT + +function cleanup() { + kill $(jobs -p) >/dev/null 2>&1 +} + +unset -v port +unset -v url + +metrics="" +while getopts l:p: flag +do + case "${flag}" in + l) location="${OPTARG}";; + p) port="${OPTARG}";; + esac +done + +if [[ -z $port ]]; then + printf "port (-p) is required\n" + exit 1 +fi + +function get_url { + # Vast.ai + if [[ $DIRECT_ADDRESS == "auto#vast-ai" ]]; then + declare -n vast_mapped_port=VAST_TCP_PORT_${port} + if [[ -n $vast_mapped_port && -n $PUBLIC_IPADDR ]]; then + url="http://${PUBLIC_IPADDR}:${vast_mapped_port}" + fi + # Runpod.io + elif [[ $DIRECT_ADDRESS == "auto#runpod-io" ]]; then + declare -n runpod_mapped_port=RUNPOD_TCP_PORT_${port} + if [[ -n $runpod_mapped_port && -n $RUNPOD_PUBLIC_IP ]]; then + url="http://${RUNPOD_PUBLIC_IP}:${runpod_mapped_port}" + elif [[ -n $RUNPOD_POD_ID ]]; then + url="https://${RUNPOD_POD_ID}-${port}.proxy.runpod.net" + fi + # Other cloud / local + else + url="http://${DIRECT_ADDRESS}:${port}" + fi + + if [[ -n $url ]]; then + printf "%s%s\n" "$url" "$location" + exit 0 + else + printf "Could not create URL\n" + exit 1 + fi +} + +get_url \ No newline at end of file diff --git a/build/COPY_ROOT/opt/ai-dock/bin/fix-permissions.sh b/build/COPY_ROOT/opt/ai-dock/bin/fix-permissions.sh index 5cc3d6b..3f97992 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/fix-permissions.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/fix-permissions.sh @@ -26,15 +26,19 @@ function fix_container() { } function fix_workspace() { - printf "Fixing workspace permissions...\n" - chown -R ${WORKSPACE_UID}.${WORKSPACE_GID} "${WORKSPACE}" - chmod -R g+s ${WORKSPACE} - setfacl -R -d -m u:"${WORKSPACE_UID}":rwx "${WORKSPACE}" - setfacl -R -d -m m:rwx "${WORKSPACE}" - chmod o-rw ${WORKSPACE}/home/user - if [[ -e ${WORKSPACE}/home/user/.ssh/authorized_keys ]]; then - chmod 700 ${WORKSPACE}/home/user/.ssh - chmod 600 ${WORKSPACE}/home/user/.ssh/authorized_keys + if [[ $WORKSPACE_PERMISSIONS == "true" ]]; then + printf "Fixing workspace permissions...\n" + chown -R ${WORKSPACE_UID}.${WORKSPACE_GID} "${WORKSPACE}" + chmod -R g+s ${WORKSPACE} + setfacl -R -d -m u:"${WORKSPACE_UID}":rwx "${WORKSPACE}" + setfacl -R -d -m m:rwx "${WORKSPACE}" + chmod o-rw ${WORKSPACE}/home/user + if [[ -e ${WORKSPACE}/home/user/.ssh/authorized_keys ]]; then + chmod 700 ${WORKSPACE}/home/user/.ssh + chmod 600 ${WORKSPACE}/home/user/.ssh/authorized_keys + fi + else + printf "No permissions changed (non-standard fs)\n" fi } diff --git a/build/COPY_ROOT/opt/ai-dock/bin/init.sh b/build/COPY_ROOT/opt/ai-dock/bin/init.sh index 7c39cae..61a44c5 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/init.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/init.sh @@ -122,19 +122,26 @@ function init_set_ssh_keys() { } init_set_web_credentials() { + export SERVICEPORTAL_LOGIN=$(direct-url.sh -p "${SERVICEPORTAL_PORT_HOST:-1111}" -l "/login") + export SERVICEPORTAL_HOME=$(direct-url.sh -p "${SERVICEPORTAL_PORT_HOST:-1111}") + if [[ -z $WEB_USER ]]; then - WEB_USER=user + export WEB_USER=user + fi + + if [[ -z $WEB_PASSWORD ]]; then + export WEB_PASSWORD="$(openssl rand -base64 12)" fi - if [[ -z $WEB_PASSWORD && -z $WEB_PASSWORD_HASH ]]; then - WEB_PASSWORD=password - elif [[ -z $WEB_PASSWORD ]]; then - WEB_PASSWORD="********" + export WEB_PASSWORD_B64="$(printf "%s:%s" "$WEB_USER" "$WEB_PASSWORD" | base64)" + + if [[ -z $WEB_TOKEN ]]; then + # Not the same as password (probably!) + export WEB_TOKEN="$(openssl rand -base64 32)" fi - if [[ $WEB_PASSWORD != "********" ]]; then - WEB_PASSWORD_HASH=$(hash-password.sh -p $WEB_PASSWORD -r 15) - export WEB_PASSWORD="********" + if [[ -n $DISPLAY && -z $COTURN_PASSWORD ]]; then + export COTURN_PASSWORD="auto_$(openssl rand -base64 8)" fi printf "%s %s" "$WEB_USER" "$WEB_PASSWORD_HASH" > /opt/caddy/etc/basicauth @@ -156,10 +163,13 @@ function init_count_gpus() { } function init_count_quicktunnels() { - if [[ ! ${CF_QUICK_TUNNELS,,} = "true" ]]; then + if [[ ${CF_QUICK_TUNNELS,,} == "false" ]]; then export CF_QUICK_TUNNELS_COUNT=0 else - export CF_QUICK_TUNNELS_COUNT=$(grep -l "METRICS_PORT" /opt/ai-dock/bin/supervisor-*.sh | wc -l) + export CF_QUICK_TUNNELS_COUNT=$(grep -l "QUICKTUNNELS=true" /opt/ai-dock/bin/supervisor-*.sh | wc -l) + if [[ -z $TUNNEL_TRANSPORT_PROTOCOL ]]; then + export TUNNEL_TRANSPORT_PROTOCOL=http2 + fi fi } @@ -199,6 +209,17 @@ function init_set_workspace() { touch "${no_mount_warning_file}" printf "%b" "${no_mount_warning}" > "${no_mount_warning_file}" fi + # Ensure we have a proper linux filesystem so we don't run into errors on sync + if [[ $WORKSPACE_MOUNTED == "true" ]]; then + test_file=${WORKSPACE}/.ai-dock-permissions-test + touch $test_file + if chown ${WORKSPACE_UID}.${WORKSPACE_GID} $test_file; then + export WORKSPACE_PERMISSIONS=true + else + export WORKSPACE_PERMISSIONS=false + fi + rm $test_file + fi } # This is a convenience for X11 containers and bind mounts - No additional security implied. @@ -208,7 +229,7 @@ function init_create_user() { mkdir -p ${home_dir} groupadd -g $WORKSPACE_GID $USER_NAME useradd -ms /bin/bash $USER_NAME -d $home_dir -u $WORKSPACE_UID -g $WORKSPACE_GID - printf "user:%s" "$USER_PASSWORD" | chpasswd + printf "user:%s" "${USER_PASSWORD}" | chpasswd usermod -a -G $USER_GROUPS $USER_NAME # May not exist - todo check device ownership usermod -a -G render $USER_NAME @@ -252,7 +273,11 @@ function init_sync_mamba_envs() { mkdir -p ${WORKSPACE}/environments printf "Moving mamba environments to %s...\n" "${WORKSPACE}" while sleep 10; do printf "Waiting for workspace mamba sync...\n"; done & - rsync -auSHh --stats /opt/micromamba/ "${ws_mamba_target}" + if [[ $WORKSPACE_PERMISSIONS == "true" ]]; then + rsync -auSHh --stats /opt/micromamba/ "${ws_mamba_target}" + else + rsync -uSHh --stats /opt/micromamba/ "${ws_mamba_target}" + fi kill $! wait $! 2>/dev/null printf "Moved mamba environments to %s\n" "${WORKSPACE}" diff --git a/build/COPY_ROOT/opt/ai-dock/bin/set-web-credentials.sh b/build/COPY_ROOT/opt/ai-dock/bin/set-web-credentials.sh index 9e69a13..a00fab7 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/set-web-credentials.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/set-web-credentials.sh @@ -5,11 +5,14 @@ if [[ -z $2 ]]; then exit 1 fi -WEB_USER=$1 -WEB_PASSWORD_HASH=$(hash-password.sh -p $2 -r 15) -export WEB_PASSWORD="********" +export WEB_USER=$1 +env-store WEB_USER +export WEB_PASSWORD=$2 +env-store WEB_PASSWORD +export WEB_PASSWORD_B64="$(printf "%s:%s" "$WEB_USER" "$WEB_PASSWORD" | base64)" +env-store WEB_PASSWORD_B64 printf "Setting credentials and restarting proxy server...\n" -printf "%s %s" "$WEB_USER" "$WEB_PASSWORD_HASH" > /opt/caddy/etc/basicauth -supervisorctl restart caddy +supervisorctl restart serviceportal +supervisorctl restart caddy diff --git a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-caddy.sh b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-caddy.sh index 12a3944..8b27b2b 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-caddy.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-caddy.sh @@ -31,10 +31,6 @@ function start() { template_file="/opt/caddy/share/service_config" fi - if [[ ${WEB_ENABLE_AUTH,,} != 'false' && ${proxy_secure,,} != 'false' ]]; then - template_file="${template_file}_auth" - fi - cp "${template_file}" /tmp/caddy sed -i "s/!PROXY_PORT/${proxy_port}/g" /tmp/caddy sed -i "s/!LISTEN_PORT/${listen_port}/g" /tmp/caddy diff --git a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-cloudflared.sh b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-cloudflared.sh index 6be16c7..c526d82 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-cloudflared.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-cloudflared.sh @@ -5,7 +5,7 @@ trap cleanup EXIT METRICS_PORT=2999 function cleanup() { - kill $(lsof -t -i:${METRICS_PORT}) > /dev/null 2>&1 & + fuser -k -SIGTERM ${METRICS_PORT}/tcp > /dev/null 2>&1 & wait -n } @@ -20,7 +20,7 @@ function start() { printf "Starting Cloudflare daemon...\n" - kill -9 $(lsof -t -i:${METRICS_PORT}) > /dev/null 2>&1 & + fuser -k -SIGKILL ${METRICS_PORT}/tcp > /dev/null 2>&1 & wait -n cloudflared tunnel --metrics localhost:"${METRICS_PORT}" run --token "${CF_TUNNEL_TOKEN}" diff --git a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-quicktunnel.sh b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-quicktunnel.sh index c70534e..c23b3c9 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-quicktunnel.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-quicktunnel.sh @@ -3,7 +3,7 @@ trap cleanup EXIT function cleanup() { - kill $(lsof -t -i:${metrics_port}) > /dev/null 2>&1 & + fuser -k -SIGTERM ${metrics_port}/tcp > /dev/null 2>&1 & wait -n } @@ -31,7 +31,7 @@ function start() { fi # Ensure the port is available (kill stale for restart) - kill -9 $(lsof -t -i:${metrics_port}) > /dev/null 2>&1 & + fuser -k -SIGKILL ${metrics_port}/tcp > /dev/null 2>&1 & wait -n cloudflared tunnel ${metrics} ${tunnel} diff --git a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-serviceportal.sh b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-serviceportal.sh index 5169df3..f64db13 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-serviceportal.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-serviceportal.sh @@ -2,16 +2,16 @@ trap cleanup EXIT -LISTEN_PORT=${SERVICEPORTAL_PORT_LOCAL:-11111} +LISTEN_PORT=11111 METRICS_PORT=${SERVICEPORTAL_METRICS_PORT:-21111} PROXY_PORT=${SERVICEPORTAL_PORT_HOST:-1111} -# Auth is true for defined paths - See /opt/caddy/share/service_config_11111_auth -PROXY_SECURE=true +QUICKTUNNELS=true + SERVICE_NAME="Service Portal" function cleanup() { rm /run/http_ports/$PROXY_PORT > /dev/null 2>&1 - kill $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 & + fuser -k -SIGTERM ${LISTEN_PORT}/tcp > /dev/null 2>&1 & wait -n } @@ -37,7 +37,7 @@ function start() { printf "Starting ${SERVICE_NAME}...\n" - kill -9 $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 & + fuser -k -SIGKILL ${LISTEN_PORT}/tcp > /dev/null 2>&1 & wait -n /usr/bin/python3 /opt/ai-dock/fastapi/serviceportal/main.py \ diff --git a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-sshd.sh b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-sshd.sh index f1616b1..83c4a55 100755 --- a/build/COPY_ROOT/opt/ai-dock/bin/supervisor-sshd.sh +++ b/build/COPY_ROOT/opt/ai-dock/bin/supervisor-sshd.sh @@ -3,7 +3,7 @@ trap cleanup EXIT function cleanup() { - kill $(lsof -t -i:${SSH_PORT}) > /dev/null 2>&1 & + fuser -k -SIGTERM 22/tcp > /dev/null 2>&1 & wait -n } @@ -15,11 +15,6 @@ function start() { exec sleep 6 fi - # Support previous config - if [[ ! -v SSH_PORT || -z $SSH_PORT ]]; then - SSH_PORT=${SSH_PORT_LOCAL:-22} - fi - ak_file="/root/.ssh/authorized_keys" if [[ ! $(ssh-keygen -l -f $ak_file) ]]; then printf "Skipping SSH server: No public key\n" 1>&2 @@ -35,11 +30,11 @@ function start() { printf "Starting SSH server on port ${SSH_PORT}...\n" - kill -9 $(lsof -t -i:$SSH_PORT) > /dev/null 2>&1 & + fuser -k -SIGKILL 22/tcp > /dev/null 2>&1 & wait -n /usr/bin/ssh-keygen -A - /usr/sbin/sshd -D -p $SSH_PORT + /usr/sbin/sshd -D -p 22 } start 2>&1 \ No newline at end of file