Base improvements

Add timezone Add runtime user Fix permissions Drop rclone mount support - Not supported in cloud and encourages unhealthy behaviour locally - Mount outside and bind Add additional archive tools Rationale: Prevent major base deviation between standard and X11 desktop variants
ai-dock · Jan 1, 2024 · 9456fdc · 9456fdc
1 parent 60c8d9f
commit 9456fdc
Show file tree

Hide file tree

Showing 9 changed files with 150 additions and 179 deletions.
diff --git a/README.md b/README.md
@@ -108,7 +108,6 @@ You can use the included `cloudflared` service to make secure connections withou
 | `GPU_COUNT`              | Limit the number of available GPUs |
 | `PROVISIONING_SCRIPT`    | URL of a remote script to execute on init. See [note](#provisioning-script). |
 | `RCLONE_*`               | Rclone configuration - See [rclone documentation](https://rclone.org/docs/#config-file) |
-| `SKIP_ACL`               | Set `true` to skip modifying workspace ACL |
 | `SSH_PORT_LOCAL`         | Set a non-standard port for SSH (default `22`) |
 | `SSH_PUBKEY`             | Your public key for SSH |
 | `WEB_ENABLE_AUTH`        | Enable password protection for web services (default `true`) |
@@ -125,7 +124,19 @@ Example usage: `docker run -e STANDARD_VAR1="this value" -e STANDARD_VAR2="that
 
 ## Security
 
-By default, all exposed web services other than the port redirect page are protected by HTTP basic authentication.
+All ai-dock containers are interactive and will not drop root privileges. You should ensure that your docker daemon runs as an unprivileged user.
+
+### System
+
+A system user will be created at startup. The UID will be either 1000 or will match the UID of the `$WORKSPACE` bind mount.
+
+No password is set for this user and it will share the root user's ssh public key.
+
+Some processes may start in the user context for convenience only.
+
+### Web Services
+
+By default, all exposed web services other than the port redirect index page are protected by HTTP basic authentication.
 
 The default username is `user` and the password is `password`.
 
@@ -203,8 +214,6 @@ As docker containers generally run as the root user, new files created in /works
 
 To ensure that the files remain accessible to the local user that owns the directory, the docker entrypoint will set a default ACL on the directory by executing the commamd `setfacl -d -m u:${WORKSPACE_UID}:rwx /workspace`.
 
-If you do not want this, you can set the environment variable `SKIP_ACL=true`.
-
 ## Running Services
 
 This image will spawn multiple processes upon starting a container because some of our remote environments do not support more than one container per instance.
@@ -275,30 +284,6 @@ See [this guide](https://link.ai-dock.org/guide-sshd-do) by DigitalOcean for an
 >[!NOTE]  
 >_SSHD is included because the end-user should be able to know the version prior to deloyment. Using a providers add-on, if available, does not guarantee this._
 
-### Rclone mount
-
-Rclone allows you to access your cloud storage from within the container by configuring one or more remotes. If you are unfamiliar with the project you can find out more at the [Rclone website](https://rclone.org/).
-
-Any Rclone remotes that you have specified, either through mounting the config directory or via setting environment variables will be mounted at `/workspace/remote/[remote name]`. For this service to start, the following conditions must be met:
-
-- Fuse3 installed in the host operating system
-- Kernel module `fuse` loaded in the host
-- Host `/etc/passwd` mounted in the container
-- Host `/etc/group` mounted in the container
-- Host device `/dev/fuse` made available to the container
-- Container must run with `cap-add SYS_ADMIN`
-- Container must run with `securiry-opt apparmor:unconfined`
-- At least one remote must be configured
-
-The provided docker-compose.yaml includes a working configuration (add your own remotes).
-
-In the event that the conditions listed cannot be met, `rclone` will still be available to use via the CLI - only mounts will be unavailable.
-
-If you intend to use the `rclone create` command to interactively generate remote configurations you should ensure port `53682` is accessible. See https://rclone.org/remote_setup/ for further details.
-
->[!NOTE]  
->_Rclone is included to give the end-user an opportunity to easily transfer files between the instance and their cloud storage provider._
-
 >[!WARNING]  
 >You should only provide auth tokens in secure cloud environments.
 

diff --git a/build/COPY_ROOT/etc/supervisor/supervisord/conf.d/rclone_mount.conf b/build/COPY_ROOT/etc/supervisor/supervisord/conf.d/rclone_mount.conf
diff --git a/build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh b/build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh
@@ -4,44 +4,69 @@ export MAMBA_CREATE="micromamba create --always-softlink -y -c conda-forge"
 export MAMBA_INSTALL="micromamba install --always-softlink -y -c conda-forge"
 printf "export MAMBA_CREATE=\"%s\"\n" "${MAMBA_CREATE}" >> /opt/ai-dock/etc/environment.sh
 printf "export MAMBA_INSTALL=\"%s\"\n" "${MAMBA_INSTALL}" >> /opt/ai-dock/etc/environment.sh
+printf "git config --global --add safe.directory \"*\"\n" >> /opt/ai-dock/etc/environment.sh
 
+dpkg --add-architecture i386
 apt-get update
 apt-get upgrade -y --no-install-recommends
 
 # System packages
 $APT_INSTALL \
     acl \
+    apt-transport-https \
+    apt-utils \
     bc \
     build-essential \
     bzip2 \
     ca-certificates \
     curl \
+    dnsutils \
     dos2unix \
+    fakeroot \
+    file \
     fuse3 \
     git \
     git-lfs \
+    gnupg \
     gpg \
+    gzip \
+    htop \
     inotify-tools \
     jq \
+    language-pack-en \
     less \
     libcap2-bin \
     libelf1 \
     libglib2.0-0 \
+    locales \
     lsb-release \
     lsof \
+    mlocate \
+    net-tools \
     nano \
     openssh-server \
+    pkg-config \
     python3-pip \
+    rar \
     rclone \
     rsync \
     screen \
+    software-properties-common \
+    ssl-cert \
+    sudo \
     supervisor \
     tmux \
+    tzdata \
+    unar \
+    unrar \
     unzip \
     vim \
     wget \
     xz-utils \
-    zip
+    zip \
+    zstd
+
+  locale-gen en_US.UTF-8
 
   # These libraries are needed to run the log/redirect interfaces
   # They are needed before micromamba is guaranteed to be ready
@@ -74,8 +99,7 @@ touch /root/.ssh/authorized_keys
 chmod 600 /root/.ssh/authorized_keys
 
 # Remove less relevant parts of motd
-rm /etc/update-motd.d/10-help-text
-rm /etc/update-motd.d/60-unminimize
+rm -f /etc/update-motd.d/10-help-text
 
 # Install micromamba (conda replacement)
 mkdir -p /opt/micromamba
@@ -91,11 +115,10 @@ mkdir -p /var/empty
 mkdir -p /etc/rclone
 touch /etc/rclone/rclone.conf
 
-# Git config
-
-git config --global --add safe.directory "*"
-
 # Ensure correct environment for child builds
 
 printf "source /opt/ai-dock/etc/environment.sh\n" >> /etc/profile.d/02-ai-dock.sh
-printf "source /opt/ai-dock/etc/environment.sh\n" >> /etc/bash.bashrc
+printf "source /opt/ai-dock/etc/environment.sh\n" >> /etc/bash.bashrc
+
+# Give our runtime user full access (added to users group)
+/opt/ai-dock/bin/fix-permissions.sh -o container
diff --git a/build/COPY_ROOT/opt/ai-dock/bin/fix-permissions.sh b/build/COPY_ROOT/opt/ai-dock/bin/fix-permissions.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+function main() {
+  while getopts o: flag; do
+      case "${flag}" in
+          o) only="${OPTARG}"
+      esac
+  done
+
+  if [[ ${only,,} == "container" ]]; then
+      fix_container
+  elif [[ ${only,,} == "workspace" ]]; then
+      fix_workspace
+  else
+      fix_container
+      fix_workspace
+  fi
+}
+
+function fix_container() {
+    printf "Fixing container permissions...\n"
+    items=micromamba:$OPT_SYNC
+    IFS=: read -r -d '' -a path_array < <(printf '%s:\0' "$items")
+    for item in "${path_array[@]}"; do
+        if [[ -n $item ]]; then
+            opt_dir="/opt/${item}"
+            chown -R root.users "$opt_dir"
+            chmod -R g+s "$opt_dir"
+            chmod -R ug+rw "$opt_dir"
+            setfacl -R -d -m g:users:rwx "$opt_dir"
+            setfacl -R -d -m m:rwx "$opt_dir"
+        fi
+    done
+
+}
+
+function fix_workspace() {
+    printf "Fixing workspace permissions...\n"
+    chown -R ${WORKSPACE_UID}.${WORKSPACE_GID} "${WORKSPACE}"
+    chmod -R g+s ${WORKSPACE}
+    setfacl -R -d -m u:"${WORKSPACE_UID}":rwx "${WORKSPACE}"
+    setfacl -R -d -m m:rwx "${WORKSPACE}"
+    chmod o-rw ${WORKSPACE}/home/user
+    if [[ -e ${WORKSPACE}/home/user/.ssh/authorized_keys ]]; then
+        chmod 700 ${WORKSPACE}/home/user/.ssh
+        chmod 600 ${WORKSPACE}/home/user/.ssh/authorized_keys
+    fi
+}
+
+main "$@"
diff --git a/build/COPY_ROOT/opt/ai-dock/bin/init.sh b/build/COPY_ROOT/opt/ai-dock/bin/init.sh
@@ -19,9 +19,9 @@ function init_main() {
     init_set_web_credentials
     init_direct_address
     init_set_workspace
+    init_create_user
     init_count_gpus
     init_count_quicktunnels
-    init_count_rclone_remotes
     init_set_cf_tunnel_wanted
     touch /run/container_config
     touch /run/workspace_sync
@@ -51,9 +51,7 @@ init_serverless() {
   init_set_envs "$@"
   touch "${WORKSPACE}.update_lock"
   export CF_QUICK_TUNNELS_COUNT=0
-  export RCLONE_MOUNT_COUNT=0
   export SUPERVISOR_START_CLOUDFLARED=0
-  init_direct_address
   init_set_workspace
   init_count_gpus
   init_create_directories
@@ -159,11 +157,18 @@ function init_set_workspace() {
         export WORKSPACE=${ws_tmp//\/\//\/}
     fi
 
+    WORKSPACE_UID=$(stat -c '%u' "$WORKSPACE")
+    if [[ $WORKSPACE_UID -eq 0 ]]; then
+        WORKSPACE_UID=1000
+    fi
+    export WORKSPACE_UID
+    WORKSPACE_GID=$(stat -c '%g' "$WORKSPACE")
+    export WORKSPACE_GID
+
     if [[ -f "${WORKSPACE}".update_lock ]]; then
         export AUTO_UPDATE=false
     fi
 
-    mkdir -p "${WORKSPACE}"remote/.cache
     mkdir -p "${WORKSPACE}"storage
 
     # Determine workspace mount status
@@ -179,6 +184,29 @@ function init_set_workspace() {
     fi
 }
 
+# This is a convenience for X11 containers and bind mounts - No additional security implied.
+# These are interactive containers; root will always be available. Secure your daemon.
+function init_create_user() {
+    user_name=user
+    home_dir=${WORKSPACE}/home/user
+    mkdir -p ${home_dir}
+    groupadd -g $WORKSPACE_GID $user_name
+    useradd -ms /bin/bash $user_name -d $home_dir -u $WORKSPACE_UID -g $WORKSPACE_GID
+    usermod -a -G users,sudo,audio,video,render,adm,cdrom,input,lp,lpadmin,plugdev,pulse-access,scanner,ssl-cert,tty,voice $user_name
+    ln -s $home_dir /home/${user_name}
+    echo "${user_name} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+    if [[ ! -e ${home_dir}/.bashrc ]]; then
+        cp -f /root/.bashrc ${home_dir}
+        chown ${WORKSPACE_UID}:${WORKSPACE_GID} ${home_dir}/.bashrc
+    fi
+    # Set initial keys to match root
+    if [[ -e /root/.ssh/authorized_keys  && ! -d ${home_dir}/.ssh ]]; then
+        mkdir -m 700 ${home_dir}/.ssh
+        cp /root/.ssh/authorized_keys ${home_dir}/.ssh
+        chmod 600 ${home_dir}/.ssh/authorized_keys
+    fi
+}
+
 function init_sync_mamba_envs() {
     printf "Mamba sync start: %s\n" "$(date +"%x %T.%3N")" >> /var/log/timing_data
     ws_mamba_target="${WORKSPACE}environments/micromamba-${IMAGE_SLUG}"
@@ -285,16 +313,8 @@ init_sync_opt() {
 }
 
 init_set_workspace_permissions() {
-    # Ensure the workspace owner can access files from outside of the container
-    WORKSPACE_UID=$(stat -c '%u' "$WORKSPACE")
-    export WORKSPACE_UID
-    WORKSPACE_GID=$(stat -c '%g' "$WORKSPACE")
-    export WORKSPACE_GID
-    if [[ ${WORKSPACE_SYNC,,} != 'false' && ${SKIP_ACL,,} != 'false' && $WORKSPACE_UID -gt 0 ]]; then
-        setfacl -R -d -m u:"${WORKSPACE_UID}":rwx "${WORKSPACE}"
-        setfacl -R -d -m m:rwx "${WORKSPACE}"
-        chown -R ${WORKSPACE_UID}.${WORKSPACE_GID} "${WORKSPACE}"
-    fi
+    # Ensure the workspace owner/container user can access files from outside of the container
+    /opt/ai-dock/bin/fix-permissions.sh -o workspace
 }
 
 function init_set_cf_tunnel_wanted() {
@@ -305,34 +325,9 @@ function init_set_cf_tunnel_wanted() {
     fi
 }
 
-function init_count_rclone_remotes() {
-    # Determine if rclone mount will be possible
-    mount_env_warning_file="${WORKSPACE}remote/WARNING-CANNOT-MOUNT-REMOTES.txt"
-    no_remotes_warning_file="${WORKSPACE}remote/WARNING-NO-REMOTES-CONFIGURED.txt"
-    rm ${mount_env_warning_file} > /dev/null 2>&1
-    rm ${no_remotes_warning_file} > /dev/null 2>&1
-    capsh --print | grep "Current:" | grep -q cap_sys_admin
-    if [[ $? -ne 0 || ! -e /dev/fuse ]]; then
-        # Not in container with sufficient privileges
-        mount_env_warning="Environment unsuitable for rclone mount...\nrclone remains available via CLI\n"
-        printf "%b" "${mount_env_warning}"
-        touch "${mount_env_warning_file}"
-        printf "%b" "${mount_env_warning}" > "${mount_env_warning_file}"
-        export RCLONE_MOUNT_COUNT=0
-    else
-        RCLONE_MOUNT_COUNT=$(rclone listremotes |wc -w)
-        export RCLONE_MOUNT_COUNT
-        if [[ $RCLONE_MOUNT_COUNT -eq 0 ]]; then
-            no_remotes_warning="You have no configured rclone remotes to be mounted\n"
-            printf "%b" "${no_remotes_warning}"
-            touch "${no_remotes_warning_file}"
-            printf "%b" "${no_remotes_warning}" > ${no_remotes_warning_file}
-        fi
-    fi
-}
-
 function init_direct_address() {
-    # Ensure set
+    export EXTERNAL_IP_ADDRESS="$(dig +short myip.opendns.com @resolver1.opendns.com)"
+
     if [[ ! -v DIRECT_ADDRESS ]]; then
         DIRECT_ADDRESS=""
     fi
@@ -341,7 +336,7 @@ function init_direct_address() {
         export DIRECT_ADDRESS=""
     elif [[ -z $DIRECT_ADDRESS || ${DIRECT_ADDRESS_GET_WAN,,} == 'true' ]]; then
         if [[ ${DIRECT_ADDRESS_GET_WAN,,} == 'true' ]]; then
-            export DIRECT_ADDRESS="$(curl https://icanhazip.com)"
+            export DIRECT_ADDRESS="$EXTERNAL_IP_ADDRESS"
         # Detected provider has direct connection method
         elif env | grep 'VAST' > /dev/null 2>&1; then
             export DIRECT_ADDRESS="auto#vast-ai"
@@ -391,15 +386,21 @@ function init_source_preflight_script() {
 
 function init_write_environment() {
     # Ensure all variables available for interactive sessions
+    skip_keys=(
+        "HOME"
+    )
     while IFS='=' read -r -d '' key val; do
-        printf "export %s=\"%s\"\n" "$key" "$val" >> /opt/ai-dock/etc/environment.sh
+        if [[ ! ${skip_keys[@]} =~ "$key" ]]; then
+            printf "export %s=\"%s\"\n" "$key" "$val" >> /opt/ai-dock/etc/environment.sh
+        fi
     done < <(env -0)
 
     if [[ -n $MAMBA_DEFAULT_ENV ]]; then
       printf "micromamba activate %s\n" $MAMBA_DEFAULT_ENV >> /root/.bashrc
     fi
 
     printf "cd %s\n" "$WORKSPACE" >> /root/.bashrc
+    ln -snf "/usr/share/zoneinfo/$TZ" /etc/localtime && echo "$TZ" | sudo tee /etc/timezone > /dev/null
 }
 
 function init_get_provisioning_script() {