Merge branch 'iot-salzburg:master' into master

.build/10activate-conda-env.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+# This registers the initialization code for the conda shell code
+# It also activates default environment in the end, so we don't need to activate it manually
+# Documentation: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html
+eval "$(conda shell.bash hook)"

.build/Rprofile.site

@@ -0,0 +1,4 @@
+# Add R mimetype to specify how the plot returns from R to the browser.
+# https://notebook.community/andrie/jupyter-notebook-samples/Changing%20R%20plot%20options%20in%20Jupyter
+
+options(jupyter.plot_mimetypes = c('text/plain', 'image/png', 'image/jpeg', 'image/svg+xml', 'application/pdf'))

.build/docker_healthcheck.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import json
+import os
+from pathlib import Path
+
+import requests
+
+# Several operations below deliberately don't check for possible errors
+# As this is a healthcheck, it should succeed or raise an exception on error
+
+runtime_dir = Path("/home/") / os.environ["NB_USER"] / ".local/share/jupyter/runtime/"
+json_file = next(runtime_dir.glob("*server-*.json"))
+
+url = json.loads(json_file.read_bytes())["url"]
+url = url + "api"
+
+proxies = {
+    "http": "",
+    "https": "",
+}
+
+r = requests.get(url, proxies=proxies, verify=False)  # request without SSL verification
+r.raise_for_status()
+print(r.content)

.build/fix-permissions

@@ -1,16 +1,14 @@
 #!/bin/bash
-# set permissions on a directory
-# after any installation, if a directory needs to be (human) user-writable,
-# run this script on it.
-# It will make everything in the directory owned by the group ${NB_GID}
-# and writable by that group.
+# Set permissions on a directory
+# After any installation, if a directory needs to be (human) user-writable, run this script on it.
+# It will make everything in the directory owned by the group ${NB_GID} and writable by that group.
 # Deployments that want to set a specific user id can preserve permissions
 # by adding the `--group-add users` line to `docker run`.
 
-# uses find to avoid touching files that already have the right permissions,
-# which would cause massive image explosion
+# Uses find to avoid touching files that already have the right permissions,
+# which would cause a massive image explosion
 
-# right permissions are:
+# Right permissions are:
 # group=${NB_GID}
 # AND permissions include group rwX (directory-execute)
 # AND directories have setuid,setgid bits set
@@ -23,13 +21,13 @@ for d in "$@"; do
             -group "${NB_GID}" \
             -a -perm -g+rwX \
         \) \
-        -exec chgrp "${NB_GID}" {} \; \
-        -exec chmod g+rwX {} \;
+        -exec chgrp "${NB_GID}" -- {} \+ \
+        -exec chmod g+rwX -- {} \+
     # setuid, setgid *on directories only*
     find "${d}" \
         \( \
             -type d \
             -a ! -perm -6000 \
         \) \
-        -exec chmod +6000 {} \;
+        -exec chmod +6000 -- {} \+
 done

.build/jupyter_server_config.py

@@ -4,14 +4,17 @@
 import os
 import stat
 import subprocess
+from pathlib import Path
 
 from jupyter_core.paths import jupyter_data_dir
 
 c = get_config()  # noqa: F821
 c.ServerApp.ip = "0.0.0.0"
-c.ServerApp.port = 8888
 c.ServerApp.open_browser = False
 
+# to output both image/svg+xml and application/pdf plot formats in the notebook file
+c.InlineBackend.figure_formats = {"png", "jpeg", "svg", "pdf"}
+
 # https://github.com/jupyter/notebook/issues/3130
 c.FileContentsManager.delete_to_trash = False
 
@@ -22,17 +25,16 @@
 [req_distinguished_name]
 """
 if "GEN_CERT" in os.environ:
-    dir_name = jupyter_data_dir()
-    pem_file = os.path.join(dir_name, "notebook.pem")
-    os.makedirs(dir_name, exist_ok=True)
+    dir_name = Path(jupyter_data_dir())
+    dir_name.mkdir(parents=True, exist_ok=True)
+    pem_file = dir_name / "notebook.pem"
 
     # Generate an openssl.cnf file to set the distinguished name
-    cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf")
-    if not os.path.isfile(cnf_file):
-        with open(cnf_file, "w") as fh:
-            fh.write(OPENSSL_CONFIG)
+    cnf_file = Path(os.getenv("CONDA_DIR", "/usr/lib")) / "ssl/openssl.cnf"
+    if not cnf_file.exists():
+        cnf_file.write_text(OPENSSL_CONFIG)
 
-    # Generate a certificate if one doesn't exist on disk
+    # Generate a certificate if one doesn't exist on a disk
     subprocess.check_call(
         [
             "openssl",
@@ -48,10 +50,9 @@
         ]
     )
     # Restrict access to the file
-    os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR)
-    c.ServerApp.certfile = pem_file
+    pem_file.chmod(stat.S_IRUSR | stat.S_IWUSR)
+    c.ServerApp.certfile = str(pem_file)
 
-# Change default umask for all subprocesses of the notebook server if set in
-# the environment
+# Change default umask for all subprocesses of the Server if set in the environment
 if "NB_UMASK" in os.environ:
     os.umask(int(os.environ["NB_UMASK"], 8))

.build/jupyter_server_config_token_addendum.py

@@ -0,0 +1,5 @@
+
+# Set static token
+import os
+if os.getenv("JUPYTER_TOKEN"):
+    c.ServerApp.token = os.getenv("JUPYTER_TOKEN")

.build/run-hooks.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+# The run-hooks.sh script looks for *.sh scripts to source
+# and executable files to run within a passed directory
+
+if [ "$#" -ne 1 ]; then
+    echo "Should pass exactly one directory"
+    return 1
+fi
+
+if [[ ! -d "${1}" ]]; then
+    echo "Directory ${1} doesn't exist or is not a directory"
+    return 1
+fi
+
+echo "Running hooks in: ${1} as uid: $(id -u) gid: $(id -g)"
+for f in "${1}/"*; do
+    # Handling a case when the directory is empty
+    [ -e "${f}" ] || continue
+    case "${f}" in
+        *.sh)
+            echo "Sourcing shell script: ${f}"
+            # shellcheck disable=SC1090
+            source "${f}"
+            # shellcheck disable=SC2181
+            if [ $? -ne 0 ]; then
+                echo "${f} has failed, continuing execution"
+            fi
+            ;;
+        *)
+            if [ -x "${f}" ]; then
+                echo "Running executable: ${f}"
+                "${f}"
+                # shellcheck disable=SC2181
+                if [ $? -ne 0 ]; then
+                    echo "${f} has failed, continuing execution"
+                fi
+            else
+                echo "Ignoring non-executable: ${f}"
+            fi
+            ;;
+    esac
+done
+echo "Done running hooks in: ${1}"

.build/setup-scripts/activate_notebook_custom_env.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import json
+import os
+import sys
+from pathlib import Path
+
+env_name = sys.argv[1]
+CONDA_DIR = os.environ["CONDA_DIR"]
+
+file = Path.home() / f".local/share/jupyter/kernels/{env_name}/kernel.json"
+content = json.loads(file.read_text())
+content["env"] = {
+    "XML_CATALOG_FILES": "",
+    "PATH": f"{CONDA_DIR}/envs/{env_name}/bin:$PATH",
+    "CONDA_PREFIX": f"{CONDA_DIR}/envs/{env_name}",
+    "CONDA_PROMPT_MODIFIER": f"({env_name}) ",
+    "CONDA_SHLVL": "2",
+    "CONDA_DEFAULT_ENV": env_name,
+    "CONDA_PREFIX_1": CONDA_DIR,
+}
+
+file.write_text(json.dumps(content, indent=1))

.build/setup-scripts/setup-julia-packages.bash

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -exuo pipefail
+# Requirements:
+# - Run as a non-root user
+# - The JULIA_PKGDIR environment variable is set
+# - Julia is already set up, with the setup-julia.bash command
+
+
+# If we don't specify what CPUs the precompilation should be done for, it's
+# *only* done for the target of the host doing the compilation.  When the
+# container runs on a host that's the same architecture, but a *different*
+# generation of CPU than what the build host was, the precompilation is useless
+# and Julia takes a long long time to start up. This specific multitarget comes
+# from https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L20-L76,
+# and may need to be updated as new CPU generations come out.
+# If the architecture the container runs on is different,
+# precompilation may still have to be re-done on first startup - but this
+# *should* catch most of the issues.  See
+# https://github.com/jupyter/docker-stacks/issues/2015 for more information
+if [ "$(uname -m)" == "x86_64" ]; then
+    # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L24
+    # for an explanation of these options
+    export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)"
+elif [ "$(uname -m)" == "aarch64" ]; then
+    # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L54
+    # for an explanation of these options
+    export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel"
+fi
+
+# Install base Julia packages
+julia -e '
+import Pkg;
+Pkg.update();
+Pkg.add([
+    "HDF5",
+    "IJulia",
+    "Pluto"
+]);
+Pkg.precompile();
+'
+
+# Move the kernelspec out of ${HOME} to the system share location.
+# Avoids problems with runtime UID change not taking effect properly
+# on the .local folder in the jovyan home dir.
+mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/"
+chmod -R go+rx "${CONDA_DIR}/share/jupyter"
+rm -rf "${HOME}/.local"
+fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter"
+
+# Install jupyter-pluto-proxy to get Pluto to work on JupyterHub
+mamba install --yes \
+    'jupyter-pluto-proxy' && \
+    mamba clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"

.build/setup-scripts/setup-julia.bash

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -exuo pipefail
+# Requirements:
+# - Run as the root user
+# - The JULIA_PKGDIR environment variable is set
+
+# Default julia version to install if env var is not set
+# Check https://julialang.org/downloads/
+JULIA_VERSION="${JULIA_VERSION:-1.9.1}"
+
+# Figure out what architecture we are installing in
+JULIA_ARCH=$(uname -m)
+JULIA_SHORT_ARCH="${JULIA_ARCH}"
+if [ "${JULIA_SHORT_ARCH}" == "x86_64" ]; then
+    JULIA_SHORT_ARCH="x64"
+fi
+
+# Figure out Julia Installer URL
+JULIA_INSTALLER="julia-${JULIA_VERSION}-linux-${JULIA_ARCH}.tar.gz"
+JULIA_MAJOR_MINOR=$(echo "${JULIA_VERSION}" | cut -d. -f 1,2)
+
+# Download and install Julia
+cd /tmp
+mkdir "/opt/julia-${JULIA_VERSION}"
+wget --progress=dot:giga "https://julialang-s3.julialang.org/bin/linux/${JULIA_SHORT_ARCH}/${JULIA_MAJOR_MINOR}/${JULIA_INSTALLER}"
+tar xzf "${JULIA_INSTALLER}" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1
+rm "${JULIA_INSTALLER}"
+
+# Link Julia installed version to /usr/local/bin, so julia launches it
+ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
+
+# Tell Julia where conda libraries are
+mkdir -p /etc/julia
+echo "push!(Libdl.DL_LOAD_PATH, \"${CONDA_DIR}/lib\")" >> /etc/julia/juliarc.jl
+
+# Create JULIA_PKGDIR, where user libraries are installed
+mkdir "${JULIA_PKGDIR}"
+chown "${NB_USER}" "${JULIA_PKGDIR}"
+fix-permissions "${JULIA_PKGDIR}"