From e377245e45c8eac7fcd20d7a1f0a3ae445e6fdf6 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 12:57:24 +0530 Subject: [PATCH 01/22] feat: added additional dependencies --- .../2.1.1-base-python-3.11.5/Dockerfile | 135 ++++++++++++++++++ .../2.1.1-base-python-3.11.5/README.md | 81 +++++++++++ .../2.1.1-base-python-3.11.5/TUNNELING.md | 65 +++++++++ .../jupyter_codeserver_proxy/__init__.py | 28 ++++ .../icons/vs_code_icon.svg | 41 ++++++ .../codeserver-proxy/setup.py | 18 +++ .../2.1.1-base-python-3.11.5/requirements.txt | 29 ++++ .../2.1.1-base-python-3.11.5/spark/config.ini | 3 + .../vscode-tunnel/scripts/init-or-start.js | 111 ++++++++++++++ .../vscode-tunnel/scripts/refresh.sh | 1 + .../vscode-tunnel/scripts/start.sh | 1 + .../vscode-tunnel/scripts/status.sh | 1 + .../vscode-tunnel/scripts/stop.sh | 1 + .../vscode-tunnel/scripts/tunnel.sh | 1 + 14 files changed, 516 insertions(+) create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/TUNNELING.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/__init__.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/setup.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/spark/config.ini create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/init-or-start.js create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/refresh.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/start.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/status.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/stop.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/tunnel.sh diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile new file mode 100644 index 00000000..f7a6cbb6 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -0,0 +1,135 @@ +FROM python:3.11.5-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # Set up Docker repository + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + libpq-dev \ + libssl-dev \ + libffi-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Install pyenv +RUN curl https://pyenv.run | bash \ + && echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bashrc \ + && echo 'eval "$(pyenv init --path)"' >> ~/.bashrc \ + && echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc + +# Install Poetry +RUN curl -sSL https://install.python-poetry.org | python3 - \ + && echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc + +# Needed for JupyterHub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# VS Code CLI +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# Force shell to bash as Peak Platform isn't respecting JupyterHub option +RUN ln -sf /bin/bash /bin/sh + +# Expose port and set the default command +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md new file mode 100644 index 00000000..a5fbb0d0 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md @@ -0,0 +1,81 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.11.5 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +Note: The image **doesn't support** [PyCharm](https://lp.jetbrains.com/projector/) unlike previous Python workspace image versions. + +## Image details +### Base image +This image uses [3.11.5-slim-bookworm](https://hub.docker.com/layers/library/python/3.11.5-slim-bookworm/images/sha256-948a09a8f42c0b2f4308ef563dc3b76de59ebfc5d83ccb7ed7a0fbbb6c5b4713?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.11.5 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.11.5 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/TUNNELING.md b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt new file mode 100644 index 00000000..7efd206e --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt @@ -0,0 +1,29 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.27.0 +peak-sdk==1.13.0 +jupysql==0.10.16 +psycopg2-binary==2.9.10 +redshift-connector==2.1.3 +snowflake-connector-python==3.12.3 +snowflake-sqlalchemy=1.6.1 +snowflake-snowpark-python==1.25.0 +pandas=2.2.3 +pyarrow=18.0.0 +boto3==1.35.65 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/spark/config.ini b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel From 99c8bb59dd03059b1c49ad1ba34ee0fe83f749b3 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 13:22:49 +0530 Subject: [PATCH 02/22] fix: synatx error --- .../ds-pack/2.1.1-base-python-3.11.5/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt index 7efd206e..1df41d14 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt @@ -22,8 +22,8 @@ jupysql==0.10.16 psycopg2-binary==2.9.10 redshift-connector==2.1.3 snowflake-connector-python==3.12.3 -snowflake-sqlalchemy=1.6.1 +snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 -pandas=2.2.3 -pyarrow=18.0.0 +pandas==2.2.3 +pyarrow==18.0.0 boto3==1.35.65 \ No newline at end of file From 5cbf75fd087bb02d01d4c114d423532f9a366b54 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 14:17:28 +0530 Subject: [PATCH 03/22] fix: env paths --- .../2.1.1-base-python-3.11.5/Dockerfile | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index f7a6cbb6..e2ba8138 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -5,7 +5,6 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # install system packages RUN apt-get update \ - # installation requirements && apt-get install -y --no-install-recommends \ apt-transport-https \ ca-certificates \ @@ -24,8 +23,6 @@ RUN apt-get update \ && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ && apt-get update \ && apt-get install -y --no-install-recommends \ - # install system packages - # jq is needed for the Peak Platform jq \ nano \ vim \ @@ -69,17 +66,16 @@ RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" # Install pyenv RUN curl https://pyenv.run | bash \ - && echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bashrc \ - && echo 'eval "$(pyenv init --path)"' >> ~/.bashrc \ - && echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc + && echo 'export PATH="/root/.pyenv/bin:$PATH"' >> /etc/bash.bashrc \ + && echo 'eval "$(pyenv init --path)"' >> /etc/bash.bashrc \ + && echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/bash.bashrc # Install Poetry RUN curl -sSL https://install.python-poetry.org | python3 - \ - && echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc + && echo 'export PATH="/root/.local/bin:$PATH"' >> /etc/bash.bashrc -# Needed for JupyterHub -RUN npm install -g configurable-http-proxy \ - && rm -rf ~/.npm +# Add pyenv and Poetry to PATH for all contexts +ENV PATH="/root/.pyenv/bin:/root/.local/bin:$PATH" # Install Python packages COPY requirements.txt /tmp/requirements.txt @@ -87,9 +83,8 @@ COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir --upgrade \ pip \ setuptools \ - wheel - -RUN pip install -r /tmp/requirements.txt + wheel \ + && pip install -r /tmp/requirements.txt # SPARK: copy over spark config COPY ./spark /usr/local/spark @@ -133,3 +128,6 @@ CMD ["jupyterhub", \ "--Spawner.http_timeout=90", \ "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ ] + +# Verify pyenv and Poetry installations +RUN pyenv --version && poetry --version From a582cbf2692320c75f6315d6dd0fbda16b671fe4 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 15:24:37 +0530 Subject: [PATCH 04/22] fix: npm error --- workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index e2ba8138..d882dee3 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -74,6 +74,11 @@ RUN curl https://pyenv.run | bash \ RUN curl -sSL https://install.python-poetry.org | python3 - \ && echo 'export PATH="/root/.local/bin:$PATH"' >> /etc/bash.bashrc + +# Needed for JupyterHub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + # Add pyenv and Poetry to PATH for all contexts ENV PATH="/root/.pyenv/bin:/root/.local/bin:$PATH" From dceeff6ab6b5f631c9104738981349834affe59c Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 17:45:07 +0530 Subject: [PATCH 05/22] fix: install --- .../2.1.1-base-python-3.11.5/Dockerfile | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index d882dee3..a3019fbf 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -1,3 +1,4 @@ +# Original content left intact FROM python:3.11.5-slim-bookworm # Set shell to be bash and fail any pipe if any step in the pipe fails @@ -64,24 +65,21 @@ RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" && ./aws/install \ && rm -rf aws awscliv2.zip -# Install pyenv +# Install pyenv and place it in /usr/bin RUN curl https://pyenv.run | bash \ - && echo 'export PATH="/root/.pyenv/bin:$PATH"' >> /etc/bash.bashrc \ - && echo 'eval "$(pyenv init --path)"' >> /etc/bash.bashrc \ - && echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/bash.bashrc + && cp -r /root/.pyenv /usr/bin/pyenv \ + && ln -s /usr/bin/pyenv/bin/pyenv /usr/bin/pyenv \ + && ln -s /usr/bin/pyenv/bin/pyenv-virtualenv /usr/bin/pyenv-virtualenv \ + && ln -s /usr/bin/pyenv/bin/pyenv-doctor /usr/bin/pyenv-doctor -# Install Poetry +# Install Poetry and move it to /usr/bin RUN curl -sSL https://install.python-poetry.org | python3 - \ - && echo 'export PATH="/root/.local/bin:$PATH"' >> /etc/bash.bashrc - + && mv /root/.local/bin/poetry /usr/bin/poetry # Needed for JupyterHub RUN npm install -g configurable-http-proxy \ && rm -rf ~/.npm -# Add pyenv and Poetry to PATH for all contexts -ENV PATH="/root/.pyenv/bin:/root/.local/bin:$PATH" - # Install Python packages COPY requirements.txt /tmp/requirements.txt @@ -133,6 +131,3 @@ CMD ["jupyterhub", \ "--Spawner.http_timeout=90", \ "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ ] - -# Verify pyenv and Poetry installations -RUN pyenv --version && poetry --version From 553673eb810af7376dd481872b55f523d0828678 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 18:56:47 +0530 Subject: [PATCH 06/22] fix: install --- .../2.1.1-base-python-3.11.5/Dockerfile | 26 +++++++++++++------ .../2.1.1-base-python-3.11.5/requirements.txt | 3 ++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index a3019fbf..f48365e0 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -65,16 +65,26 @@ RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" && ./aws/install \ && rm -rf aws awscliv2.zip -# Install pyenv and place it in /usr/bin +# Install pyenv globally RUN curl https://pyenv.run | bash \ - && cp -r /root/.pyenv /usr/bin/pyenv \ - && ln -s /usr/bin/pyenv/bin/pyenv /usr/bin/pyenv \ - && ln -s /usr/bin/pyenv/bin/pyenv-virtualenv /usr/bin/pyenv-virtualenv \ - && ln -s /usr/bin/pyenv/bin/pyenv-doctor /usr/bin/pyenv-doctor - -# Install Poetry and move it to /usr/bin + && mkdir -p /usr/local/pyenv \ + && mv /root/.pyenv/* /usr/local/pyenv \ + && rm -rf /root/.pyenv \ + && echo 'export PYENV_ROOT="/usr/local/pyenv"' >> /etc/profile.d/pyenv.sh \ + && echo 'export PATH="/usr/local/pyenv/bin:$PATH"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv init --path)"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile.d/pyenv.sh \ + && chmod -R 755 /usr/local/pyenv + +# Ensure pyenv is available to all users +ENV PYENV_ROOT=/usr/local/pyenv +ENV PATH="/usr/local/pyenv/bin:$PATH" + +# Install Poetry globally RUN curl -sSL https://install.python-poetry.org | python3 - \ - && mv /root/.local/bin/poetry /usr/bin/poetry + && mv /root/.local/bin/poetry /usr/local/bin/poetry \ + && chmod +x /usr/local/bin/poetry \ + && echo 'export PATH="/usr/local/bin:$PATH"' >> /etc/profile.d/poetry.sh # Needed for JupyterHub RUN npm install -g configurable-http-proxy \ diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt index 1df41d14..7557a5d6 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt @@ -26,4 +26,5 @@ snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 pandas==2.2.3 pyarrow==18.0.0 -boto3==1.35.65 \ No newline at end of file +boto3==1.35.65 +python-dotenv==1.0.1 From 206afdbd7a6cbeeba674c1032791d8bc2f13e3a9 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 19:42:25 +0530 Subject: [PATCH 07/22] fix: poetry install --- workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index f48365e0..dfeb86d0 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -84,7 +84,8 @@ ENV PATH="/usr/local/pyenv/bin:$PATH" RUN curl -sSL https://install.python-poetry.org | python3 - \ && mv /root/.local/bin/poetry /usr/local/bin/poetry \ && chmod +x /usr/local/bin/poetry \ - && echo 'export PATH="/usr/local/bin:$PATH"' >> /etc/profile.d/poetry.sh + && echo 'export PATH="/usr/local/bin:$PATH"' >> /etc/bash.bashrc + # Needed for JupyterHub RUN npm install -g configurable-http-proxy \ From 91191532cb7bce45c604c2c0036fcde8036b62a3 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 20:21:08 +0530 Subject: [PATCH 08/22] fix: poetry --- .../python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index dfeb86d0..2432b31d 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -82,9 +82,11 @@ ENV PATH="/usr/local/pyenv/bin:$PATH" # Install Poetry globally RUN curl -sSL https://install.python-poetry.org | python3 - \ - && mv /root/.local/bin/poetry /usr/local/bin/poetry \ - && chmod +x /usr/local/bin/poetry \ - && echo 'export PATH="/usr/local/bin:$PATH"' >> /etc/bash.bashrc + && mv /root/.local/share/pypoetry /usr/local/share/pypoetry \ + && ln -s /usr/local/share/pypoetry/venv/bin/poetry /usr/local/bin/poetry \ + && chmod -R 755 /usr/local/share/pypoetry \ + && echo 'export PATH="/usr/local/bin:$PATH"' >> /etc/profile.d/poetry.sh + # Needed for JupyterHub From ab90f99b6375feeb5f2f3058fcb1eb67d4d69dd3 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 20:49:28 +0530 Subject: [PATCH 09/22] fix: poetry --- workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index 2432b31d..f6163f40 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -85,8 +85,8 @@ RUN curl -sSL https://install.python-poetry.org | python3 - \ && mv /root/.local/share/pypoetry /usr/local/share/pypoetry \ && ln -s /usr/local/share/pypoetry/venv/bin/poetry /usr/local/bin/poetry \ && chmod -R 755 /usr/local/share/pypoetry \ - && echo 'export PATH="/usr/local/bin:$PATH"' >> /etc/profile.d/poetry.sh - + && echo 'export PATH="/usr/local/bin:$PATH"' > /etc/profile.d/poetry.sh \ + && chmod +x /etc/profile.d/poetry.sh # Needed for JupyterHub From 5b90baf195478f9c84d6efb277f66581f97279db Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 22:29:14 +0530 Subject: [PATCH 10/22] fix: poetry installation --- .../ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index f6163f40..248bc70e 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -89,6 +89,18 @@ RUN curl -sSL https://install.python-poetry.org | python3 - \ && chmod +x /etc/profile.d/poetry.sh +# Install Poetry globally +RUN curl -sSL https://install.python-poetry.org | python3 - \ + && ln -s /root/.local/bin/poetry /usr/local/bin/poetry + +# Ensure Poetry is available globally +ENV PATH="/root/.local/bin:$PATH" + +# Create an entrypoint script to ensure new users have access to Poetry +RUN echo 'export PATH="/root/.local/bin:$PATH"' >> /etc/skel/.bashrc + + + # Needed for JupyterHub RUN npm install -g configurable-http-proxy \ && rm -rf ~/.npm From 0d7044bd56bc33873aaebcbe3540c0a5c76f02e1 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Wed, 20 Nov 2024 23:04:22 +0530 Subject: [PATCH 11/22] fix: poetry --- workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index 248bc70e..789fb31e 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -91,7 +91,8 @@ RUN curl -sSL https://install.python-poetry.org | python3 - \ # Install Poetry globally RUN curl -sSL https://install.python-poetry.org | python3 - \ - && ln -s /root/.local/bin/poetry /usr/local/bin/poetry + && ln -sf /root/.local/bin/poetry /usr/local/bin/poetry + # Ensure Poetry is available globally ENV PATH="/root/.local/bin:$PATH" From a3565dad3a373e1e5e31ff571f7e8e6c5c651a80 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 10:51:42 +0530 Subject: [PATCH 12/22] fix: poetry --- .../ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 13 ------------- .../2.1.1-base-python-3.11.5/requirements.txt | 1 + 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index 789fb31e..f6163f40 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -89,19 +89,6 @@ RUN curl -sSL https://install.python-poetry.org | python3 - \ && chmod +x /etc/profile.d/poetry.sh -# Install Poetry globally -RUN curl -sSL https://install.python-poetry.org | python3 - \ - && ln -sf /root/.local/bin/poetry /usr/local/bin/poetry - - -# Ensure Poetry is available globally -ENV PATH="/root/.local/bin:$PATH" - -# Create an entrypoint script to ensure new users have access to Poetry -RUN echo 'export PATH="/root/.local/bin:$PATH"' >> /etc/skel/.bashrc - - - # Needed for JupyterHub RUN npm install -g configurable-http-proxy \ && rm -rf ~/.npm diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt index 7557a5d6..2db5ac06 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt @@ -28,3 +28,4 @@ pandas==2.2.3 pyarrow==18.0.0 boto3==1.35.65 python-dotenv==1.0.1 +poetry==1.8.4 From abe456f4506d86825c7be2f8863868bad274eb3c Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 13:00:09 +0530 Subject: [PATCH 13/22] feat: added new dwh connectors --- .../2.1.1-base-python-3.10.13/Dockerfile | 129 ++++++++++++++++++ .../2.1.1-base-python-3.10.13/README.md | 79 +++++++++++ .../2.1.1-base-python-3.10.13/TUNNELING.md | 65 +++++++++ .../jupyter_codeserver_proxy/__init__.py | 28 ++++ .../icons/vs_code_icon.svg | 41 ++++++ .../codeserver-proxy/setup.py | 18 +++ .../jupyter_projector_proxy/__init__.py | 41 ++++++ .../icons/pycharm_icon.svg | 67 +++++++++ .../projector-proxy/setup.py | 16 +++ .../requirements.txt | 21 +++ .../spark/config.ini | 3 + .../vscode-tunnel/scripts/init-or-start.js | 111 +++++++++++++++ .../vscode-tunnel/scripts/refresh.sh | 1 + .../vscode-tunnel/scripts/start.sh | 1 + .../vscode-tunnel/scripts/status.sh | 1 + .../vscode-tunnel/scripts/stop.sh | 1 + .../vscode-tunnel/scripts/tunnel.sh | 1 + .../2.1.1-base-python-3.11.5/requirements.txt | 1 + .../2.1.1-base-python-3.8.18/Dockerfile | 129 ++++++++++++++++++ .../2.1.1-base-python-3.8.18/README.md | 79 +++++++++++ .../2.1.1-base-python-3.8.18/TUNNELING.md | 65 +++++++++ .../jupyter_codeserver_proxy/__init__.py | 28 ++++ .../icons/vs_code_icon.svg | 41 ++++++ .../codeserver-proxy/setup.py | 18 +++ .../jupyter_projector_proxy/__init__.py | 41 ++++++ .../icons/pycharm_icon.svg | 67 +++++++++ .../projector-proxy/setup.py | 16 +++ .../2.1.1-base-python-3.8.18/requirements.txt | 21 +++ .../2.1.1-base-python-3.8.18/spark/config.ini | 3 + .../vscode-tunnel/scripts/init-or-start.js | 111 +++++++++++++++ .../vscode-tunnel/scripts/refresh.sh | 1 + .../vscode-tunnel/scripts/start.sh | 1 + .../vscode-tunnel/scripts/status.sh | 1 + .../vscode-tunnel/scripts/stop.sh | 1 + .../vscode-tunnel/scripts/tunnel.sh | 1 + .../2.1.1-base-python-3.9.18/Dockerfile | 129 ++++++++++++++++++ .../2.1.1-base-python-3.9.18/README.md | 79 +++++++++++ .../2.1.1-base-python-3.9.18/TUNNELING.md | 65 +++++++++ .../jupyter_codeserver_proxy/__init__.py | 28 ++++ .../icons/vs_code_icon.svg | 41 ++++++ .../codeserver-proxy/setup.py | 18 +++ .../jupyter_projector_proxy/__init__.py | 41 ++++++ .../icons/pycharm_icon.svg | 67 +++++++++ .../projector-proxy/setup.py | 16 +++ .../2.1.1-base-python-3.9.18/requirements.txt | 21 +++ .../2.1.1-base-python-3.9.18/spark/config.ini | 3 + .../vscode-tunnel/scripts/init-or-start.js | 111 +++++++++++++++ .../vscode-tunnel/scripts/refresh.sh | 1 + .../vscode-tunnel/scripts/start.sh | 1 + .../vscode-tunnel/scripts/status.sh | 1 + .../vscode-tunnel/scripts/stop.sh | 1 + .../vscode-tunnel/scripts/tunnel.sh | 1 + 52 files changed, 1873 insertions(+) create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/TUNNELING.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/setup.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/__init__.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/setup.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/spark/config.ini create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/init-or-start.js create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/refresh.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/start.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/status.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/stop.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/tunnel.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/TUNNELING.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/setup.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/__init__.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/setup.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/spark/config.ini create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/init-or-start.js create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/refresh.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/start.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/status.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/stop.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/tunnel.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/TUNNELING.md create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/setup.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/__init__.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/setup.py create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/spark/config.ini create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/init-or-start.js create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/refresh.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/start.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/status.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/stop.sh create mode 100644 workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/tunnel.sh diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile new file mode 100644 index 00000000..0cf35eb6 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile @@ -0,0 +1,129 @@ +FROM python:3.10.13-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # docker + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Needed for jupyterhub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# vscode cli +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# Install Projector & PyCharm Community +COPY projector-proxy projector-proxy +RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ + && projector --accept-license --config-directory /usr/local/share/projector \ + ide autoinstall --config-name pycharm --ide-name "PyCharm Community Edition 2021.3.1" \ + && chmod -R a=rwx /usr/local/share/projector \ + && pip install --no-cache-dir ./projector-proxy + +# force shell to bash as Peak Platform isn't respecting jupyterhub option +RUN ln -sf /bin/bash /bin/sh + +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md new file mode 100644 index 00000000..302ba30a --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md @@ -0,0 +1,79 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.10.13 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +## Image details +### Base image +This image uses [3.10.13-slim-bookworm](https://hub.docker.com/layers/library/python/3.10.13-slim-bookworm/images/sha256-fb6e9cde0d7ae6ea8cd516a76ba78dad72e6a09465a8e93fe847d63fbcebc9a6?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.10.13 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.10.13 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/TUNNELING.md b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/__init__.py b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/__init__.py new file mode 100644 index 00000000..da8ef670 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/__init__.py @@ -0,0 +1,41 @@ +import os +import shutil + +SYS_PROJECTOR_CONFIG_DIR = "/usr/local/share/projector" +USER_PROJECTOR_CONFIG_DIR = os.path.join(os.environ["HOME"], ".projector") + + +def setup_projector(): + _copy_projector_config() + return { + "command": ["projector", "run", "pycharm"], + "launcher_entry": { + "title": "PyCharm", + "icon_path": os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "icons", + "pycharm_icon.svg", + ), + }, + # default projector config runs on port 9999 + "port": 9999, + # set timeout for projector to become responsive to 60s, as it can sometimes + # take a while + "timeout": 60, + } + + +def _copy_projector_config(): + """ + Copy the projector config folder (containing a pre-downloaded copy of PyCharm + Community) from the system location in /usr/local/share to the user's home + directory. + + Project by default looks for config files in ~/.projector. Copying config folder + from /usr/local/share/ means that each user does not individually have to install + Projector and PyCharm, but can also add their own configurations if they want to + install other Jetbrains IDEs. + """ + + if not os.path.exists(USER_PROJECTOR_CONFIG_DIR): + shutil.copytree(SYS_PROJECTOR_CONFIG_DIR, USER_PROJECTOR_CONFIG_DIR) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg new file mode 100644 index 00000000..694bce26 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PyCharm_comm_64 + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/setup.py b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/setup.py new file mode 100644 index 00000000..fa667057 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/projector-proxy/setup.py @@ -0,0 +1,16 @@ +import setuptools + +setuptools.setup( + name="jupyter-projector-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=["Jupyter"], + classifiers=["Framework :: Jupyter"], + install_requires=["jupyter-server-proxy"], + entry_points={ + "jupyter_serverproxy_servers": [ + "projector = jupyter_projector_proxy:setup_projector", + ] + }, + package_data={"jupyter_projector_proxy": ["icons/*"]}, +) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt new file mode 100644 index 00000000..e758edd4 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt @@ -0,0 +1,21 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.23.0 +peak-sdk==1.0.0 +jupysql==0.10.1 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/spark/config.ini b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt index 2db5ac06..31a10fad 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt @@ -29,3 +29,4 @@ pyarrow==18.0.0 boto3==1.35.65 python-dotenv==1.0.1 poetry==1.8.4 +sqlalchemy-redshift==0.8.14 diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile new file mode 100644 index 00000000..d487f1a3 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile @@ -0,0 +1,129 @@ +FROM python:3.8.18-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # docker + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Needed for jupyterhub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# vscode cli +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# Install Projector & PyCharm Community +COPY projector-proxy projector-proxy +RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ + && projector --accept-license --config-directory /usr/local/share/projector \ + ide autoinstall --config-name pycharm --ide-name "PyCharm Community Edition 2021.3.1" \ + && chmod -R a=rwx /usr/local/share/projector \ + && pip install --no-cache-dir ./projector-proxy + +# force shell to bash as Peak Platform isn't respecting jupyterhub option +RUN ln -sf /bin/bash /bin/sh + +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md new file mode 100644 index 00000000..e85e7773 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md @@ -0,0 +1,79 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.8.18 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +## Image details +### Base image +This image uses [python:3.8.18-slim-bookworm](https://hub.docker.com/layers/library/python/3.8.18-slim-bookworm/images/sha256-05928e4bd868a725705a451f59eab0638a3509899099955bb326624a3f9bb597?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.8.18 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.8.18 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/TUNNELING.md b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/__init__.py b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/__init__.py new file mode 100644 index 00000000..da8ef670 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/__init__.py @@ -0,0 +1,41 @@ +import os +import shutil + +SYS_PROJECTOR_CONFIG_DIR = "/usr/local/share/projector" +USER_PROJECTOR_CONFIG_DIR = os.path.join(os.environ["HOME"], ".projector") + + +def setup_projector(): + _copy_projector_config() + return { + "command": ["projector", "run", "pycharm"], + "launcher_entry": { + "title": "PyCharm", + "icon_path": os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "icons", + "pycharm_icon.svg", + ), + }, + # default projector config runs on port 9999 + "port": 9999, + # set timeout for projector to become responsive to 60s, as it can sometimes + # take a while + "timeout": 60, + } + + +def _copy_projector_config(): + """ + Copy the projector config folder (containing a pre-downloaded copy of PyCharm + Community) from the system location in /usr/local/share to the user's home + directory. + + Project by default looks for config files in ~/.projector. Copying config folder + from /usr/local/share/ means that each user does not individually have to install + Projector and PyCharm, but can also add their own configurations if they want to + install other Jetbrains IDEs. + """ + + if not os.path.exists(USER_PROJECTOR_CONFIG_DIR): + shutil.copytree(SYS_PROJECTOR_CONFIG_DIR, USER_PROJECTOR_CONFIG_DIR) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg new file mode 100644 index 00000000..694bce26 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PyCharm_comm_64 + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/setup.py b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/setup.py new file mode 100644 index 00000000..fa667057 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/projector-proxy/setup.py @@ -0,0 +1,16 @@ +import setuptools + +setuptools.setup( + name="jupyter-projector-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=["Jupyter"], + classifiers=["Framework :: Jupyter"], + install_requires=["jupyter-server-proxy"], + entry_points={ + "jupyter_serverproxy_servers": [ + "projector = jupyter_projector_proxy:setup_projector", + ] + }, + package_data={"jupyter_projector_proxy": ["icons/*"]}, +) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt new file mode 100644 index 00000000..e758edd4 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt @@ -0,0 +1,21 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.23.0 +peak-sdk==1.0.0 +jupysql==0.10.1 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/spark/config.ini b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile new file mode 100644 index 00000000..b527e491 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile @@ -0,0 +1,129 @@ +FROM python:3.9.18-slim-bookworm + +# Set shell to be bash and fail any pipe if any step in the pipe fails +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# install system packages +RUN apt-get update \ + # installation requirements + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + curl \ + gnupg2 \ + # docker + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key '95C0FAF38DB3CCAD0C080A7BDC78B2DDEABC47B7' \ + && add-apt-repository 'deb http://cloud.r-project.org/bin/linux/debian bookworm-cran40/' \ + # node + && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # install system packages + # jq is needed for the Peak Platform + jq \ + nano \ + vim \ + fish \ + zsh \ + r-base=4.2.2.20221110-2 \ + r-recommended=4.2.2.20221110-2 \ + build-essential \ + sudo \ + nodejs \ + git \ + htop \ + pandoc \ + texlive \ + texlive-science \ + texlive-latex-extra \ + texlive-luatex \ + texlive-xetex \ + texlive-pictures \ + latexmk \ + lmodern \ + docker-ce \ + less \ + libxext6 \ + libxrender1 \ + libxtst6 \ + libfreetype6 \ + libxi6 \ + openssl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# install AWS CLI v2 +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ + && unzip -q awscliv2.zip \ + && ./aws/install \ + && rm -rf aws awscliv2.zip + +# Needed for jupyterhub +RUN npm install -g configurable-http-proxy \ + && rm -rf ~/.npm + +# Install Python packages +COPY requirements.txt /tmp/requirements.txt + +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel + +RUN pip install -r /tmp/requirements.txt + +# SPARK: copy over spark config +COPY ./spark /usr/local/spark + +# VS Code +COPY codeserver-proxy codeserver-proxy +RUN curl -fsSL https://code-server.dev/install.sh \ + | sh -s -- --version 4.16.1 \ + && pip install --no-cache-dir ./codeserver-proxy + +# vscode cli +RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ + && tar -xf vscode_cli.tar.gz \ + && cp ./code /usr/local/bin \ + && rm -rf vscode_cli.tar.gz + +COPY vscode-tunnel vscode-tunnel + +RUN mkdir /vscode-tunnel/logs \ + && mkdir /vscode-tunnel/docs \ + && chmod -R 777 vscode-tunnel + +RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ + && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ + && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status + +COPY TUNNELING.md /vscode-tunnel/docs/README.md + +RUN git config --global credential.helper store \ + && git config --global core.filemode false + +# Install Projector & PyCharm Community +COPY projector-proxy projector-proxy +RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ + && projector --accept-license --config-directory /usr/local/share/projector \ + ide autoinstall --config-name pycharm --ide-name "PyCharm Community Edition 2021.3.1" \ + && chmod -R a=rwx /usr/local/share/projector \ + && pip install --no-cache-dir ./projector-proxy + +# force shell to bash as Peak Platform isn't respecting jupyterhub option +RUN ln -sf /bin/bash /bin/sh + +EXPOSE 8000 +CMD ["jupyterhub", \ + "--Spawner.default_url=/lab", \ + "--Spawner.http_timeout=90", \ + "--NotebookApp.terminado_settings={'shell_command':'[/bin/bash]'}" \ + ] diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md new file mode 100644 index 00000000..c654d1b2 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md @@ -0,0 +1,79 @@ +# workspace-python-ds-pack-2.1.0-base-python-3.9.18 +This is the Python Data Science pack image made for Workspaces. +The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. +The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). + +## Image details +### Base image +This image uses [python:3.9.18-slim-bookworm](https://hub.docker.com/layers/library/python/3.9.18-slim-bookworm/images/sha256-3b4e3a36cce74c444b333a26958d65d08b0ded00869f1557faffe8d131a0bdc6?context=explore) as its base which is maintained by [the Docker Community](https://github.com/docker-library/python). + +### OS and other details +``` +Debian GNU/Linux 12 (bookworm) +Linux Kernel 5.10.186-179.751.amzn2.x86_64 +Python 3.9.18 +``` + +### Important Linux packages installed +``` +aws-cli 2.7.4 +curl 7.88.1 +git 2.39.2 +jq 1.6 +nano 7.2 +vim 9.0 +fish 3.6.0 +zsh 5.9 +R 4.3.1 +node 18.17.1 +docker 24.0.6 +htop 3.2.0 +pandoc 2.17.1.1 +less 590 +latex 3.141592653-2.6-1.40.24 +vscode-cli 1.82.2 +``` + +### Python libraries installed +``` +ipywidgets 8.1.0 +jupyter-server-proxy 4.0.0 +jupyterhub 4.0.2 +jupyterlab-git 0.42.0 +jupyterlab-lsp 4.2.0 +jupyterlab 3.6.5 +jupyterlab_widgets 3.0.8 +jupytext 1.15.1 +lckr-jupyterlab-variableinspector 3.0.9 +mypy-ls 0.5.1 +nbconvert 7.8.0 +notebook 6.5.5 +pyls-black 0.4.7 +pyls-flake 80.4.0 +pyls-isort 0.2.2 +pyls-mypy 0.1.8 +python-lsp-black 1.3.0 +python-lsp-server[all] 1.4.1 +virtualenv 20.24.5 +peak-sdk 1.0.0 +jupysql 0.10.1 +``` + +### VS Code extensions installed +``` +cweijan.vscode-database-client2 6.6.3 +``` + +### VS Code Tunneling + +The image supports creating [remote tunnels](https://code.visualstudio.com/docs/remote/tunnels). For simplicity the image comes in handy with some node/bash scripts which can be used to `initialise`, `start`, `stop`, and `restart` the remote tunnels. More info can be found [here](./TUNNELING.md). + +## Building the image +To build the image locally, run the docker build command and pass in the required build arguments: +``` +docker build . -t workflow-python-ds-pack-2.1.0-base-python-3.9.18 +``` + +## Using the image +To use the image, select it when configuring the Workspace. +If you need to install additional dependencies or add some use case specific environment variables, it can be easily extended. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/TUNNELING.md b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/TUNNELING.md new file mode 100644 index 00000000..32f42936 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/TUNNELING.md @@ -0,0 +1,65 @@ +# Tunneling with VS Code + +This README provides instructions for setting up and managing remote tunnels using this image, making it easier for you to access your workspace remotely within VS Code. + +## What is VS Code Tunneling ? + +VS Code Tunneling is a feature that allows users to connect to a remote machine, such as a desktop PC or virtual machine, via a secure tunnel. This feature securely transmits data from one network to another, allowing users to develop against any machine of their choosing from a VS Code desktop or web client without the need for SSH or HTTPS setup. For more information, please refer (this)[https://code.visualstudio.com/docs/remote/tunnels] link. + +## Initializing a Remote Tunnel + +To create a new remote tunnel for your workspace, you can use the provided bash script: + +```bash +start-tunnel +``` + +Running this script initializes and starts a new tunnel in the background, which can be accessed remotely. To ensure its functionality, a brief authentication step using GitHub is required. + +After executing the above command, you will see the following message in the console: + +``` +To grant access to the server, please log into https://github.com/login/device and use code +``` + +Follow the link provided and enter the `` displayed in the console. + +That's it! Your new, secure tunnel should now be active and accessible at the following URL: `https://vscode.dev/tunnel/workspace--0` + +## Managing Tunnels + +### Restarting a Tunnel + +If you need to restart your tunnel for smooth operation, you can use the following command: + +```bash +restart-tunnel +``` + +### Stopping a Tunnel + +To stop an active tunnel, use the following command: + +```bash +stop-tunnel +``` + +### Starting a Tunnel + +To start a previously stopped tunnel, use this command: + +```bash +start-tunnel +``` + +### Verifying the status of tunnel + +To check the status of the tunnel, use this command: + +```bash +tunnel-status +``` + +## Additional Notes + +To view the logs of any running or closed tunnel, you can refer to the `output.log` file located at `/vscode-tunnel/logs/output.log`. Similarly, any errors related to running or closed tunnels can be found in the `/vscode-tunnel/logs/error.log` file. diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 00000000..9739bd3a --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,28 @@ +import os +import subprocess + +bash_command = "code-server --install-extension cweijan.vscode-database-client2@6.6.3" + +def setup_codeserver(): + _install_vscode_extension() + return { + 'command': ['code-server', + '--auth', + 'none', + '--disable-telemetry', + '--port={port}'], + 'launcher_entry': { + 'title': 'VS Code', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'icons', 'vs_code_icon.svg') + } + } + +def _install_vscode_extension(): + # Run the Bash command and capture the output + try: + result = subprocess.check_output(bash_command, shell=True, text=True) + print("Bash command output:") + print(result) + except subprocess.CalledProcessError as e: + print("Error running the Bash command:") + print(e) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg new file mode 100644 index 00000000..37cf0a52 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/jupyter_codeserver_proxy/icons/vs_code_icon.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/setup.py b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/setup.py new file mode 100644 index 00000000..644aa275 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/codeserver-proxy/setup.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="jupyter-codeserver-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=['Jupyter'], + classifiers=['Framework :: Jupyter'], + install_requires=['jupyter-server-proxy'], + entry_points={ + 'jupyter_serverproxy_servers': [ + 'codeserver = jupyter_codeserver_proxy:setup_codeserver', + ] + }, + package_data={ + 'jupyter_codeserver_proxy': ['icons/*'] + } +) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/__init__.py b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/__init__.py new file mode 100644 index 00000000..da8ef670 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/__init__.py @@ -0,0 +1,41 @@ +import os +import shutil + +SYS_PROJECTOR_CONFIG_DIR = "/usr/local/share/projector" +USER_PROJECTOR_CONFIG_DIR = os.path.join(os.environ["HOME"], ".projector") + + +def setup_projector(): + _copy_projector_config() + return { + "command": ["projector", "run", "pycharm"], + "launcher_entry": { + "title": "PyCharm", + "icon_path": os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "icons", + "pycharm_icon.svg", + ), + }, + # default projector config runs on port 9999 + "port": 9999, + # set timeout for projector to become responsive to 60s, as it can sometimes + # take a while + "timeout": 60, + } + + +def _copy_projector_config(): + """ + Copy the projector config folder (containing a pre-downloaded copy of PyCharm + Community) from the system location in /usr/local/share to the user's home + directory. + + Project by default looks for config files in ~/.projector. Copying config folder + from /usr/local/share/ means that each user does not individually have to install + Projector and PyCharm, but can also add their own configurations if they want to + install other Jetbrains IDEs. + """ + + if not os.path.exists(USER_PROJECTOR_CONFIG_DIR): + shutil.copytree(SYS_PROJECTOR_CONFIG_DIR, USER_PROJECTOR_CONFIG_DIR) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg new file mode 100644 index 00000000..694bce26 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/jupyter_projector_proxy/icons/pycharm_icon.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PyCharm_comm_64 + + + + + + + + + + + + + + + + diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/setup.py b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/setup.py new file mode 100644 index 00000000..fa667057 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/projector-proxy/setup.py @@ -0,0 +1,16 @@ +import setuptools + +setuptools.setup( + name="jupyter-projector-server", + version="1.0", + packages=setuptools.find_packages(), + keywords=["Jupyter"], + classifiers=["Framework :: Jupyter"], + install_requires=["jupyter-server-proxy"], + entry_points={ + "jupyter_serverproxy_servers": [ + "projector = jupyter_projector_proxy:setup_projector", + ] + }, + package_data={"jupyter_projector_proxy": ["icons/*"]}, +) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt new file mode 100644 index 00000000..e758edd4 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt @@ -0,0 +1,21 @@ +ipywidgets==8.1.0 +jupyter-server-proxy==4.0.0 +jupyterhub==4.0.2 +jupyterlab-git==0.42.0 +jupyterlab-lsp==4.2.0 +jupyterlab==3.6.5 +jupyterlab_widgets==3.0.8 +jupytext==1.15.1 +lckr-jupyterlab-variableinspector==3.0.9 +mypy-ls==0.5.1 +nbconvert==7.8.0 +notebook==6.5.5 +pyls-black==0.4.7 +pyls-flake8==0.4.0 +pyls-isort==0.2.2 +pyls-mypy==0.1.8 +python-lsp-black==1.3.0 +python-lsp-server[all]==1.4.1 +virtualenv==20.23.0 +peak-sdk==1.0.0 +jupysql==0.10.1 \ No newline at end of file diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/spark/config.ini b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/spark/config.ini new file mode 100644 index 00000000..2eb378fd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/spark/config.ini @@ -0,0 +1,3 @@ +[spark.credentials] +WORKSPACEID= +TENANTCLUSTER= diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/init-or-start.js b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/init-or-start.js new file mode 100644 index 00000000..d214581c --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/init-or-start.js @@ -0,0 +1,111 @@ +/** + * The script aims to setup a vscode tunnel for the workspace. It does by starting a tunnel in the background. + * Usage - node /vscode-tunnel/scripts setup.js + */ +const { spawn } = require('child_process'); +const fs = require('fs'); + +const outputLogFilePath = '/vscode-tunnel/logs/output.log'; +const errorLogFilePath = '/vscode-tunnel/logs/error.log'; +const scriptPath = '/vscode-tunnel/scripts/tunnel.sh'; + +function sleep(milliseconds) { + const start = new Date().getTime(); + while (new Date().getTime() - start < milliseconds); +} + +function waitForDeviceVerification() { + while (true) { + console.log('Waiting for device verification...'); + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Open this link in your browser'); + if (index >= 0) { + console.log(data.substring(index)); + break; + } + // check for errors if any + const errors = fs.readFileSync(errorLogFilePath, 'utf8'); + if (errors.length) { + console.error('There was an error creating the tunnel.'); + console.error(error); + break; + } + sleep(4000); + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + break; + } + } +} + +function printOutput() { + // Read the file and print its contents + try { + const data = fs.readFileSync(outputLogFilePath, 'utf8'); + const index = data.lastIndexOf('Visual Studio Code Server'); + if (index >= 0) { + console.log(data.substring(index)); + } else { + console.log(data); + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + try { + const data = fs.readFileSync(errorLogFilePath, 'utf8'); + if (data.length) { + console.error('There was an error creating the tunnel.'); + console.error(data); + return false; + } + } catch (error) { + if (error.code === 'ENOENT') { + console.log(`File '${fileName}' not found.`); + } else { + console.error(`An error occurred: ${err}`); + } + } + + return true; +} + +function main() { + + // Open a file to store the child process's output + const outputLogFile = fs.openSync(outputLogFilePath, 'a'); + const errorLogFile = fs.openSync(errorLogFilePath, 'a'); + + // Spawn a child process to run the shell script with stdin, stdout, and stderr redirected + const childProcess = spawn('bash', [scriptPath], { + detached: true, + stdio: ['pipe', outputLogFile, errorLogFile], // Redirect output to log files + }); + + // Close the file descriptors to release them + fs.closeSync(outputLogFile); + fs.closeSync(errorLogFile); + + console.log('Starting tunnel...'); + // initial wait + sleep(3000); + const result = printOutput(); + + if (result) { + waitForDeviceVerification(); + } + + // Unref the child process to allow the Node.js process to exit + childProcess.unref(); +} + +main(); diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/refresh.sh b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/refresh.sh new file mode 100644 index 00000000..dd1f76cd --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/refresh.sh @@ -0,0 +1 @@ +code tunnel restart diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/start.sh b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/start.sh new file mode 100644 index 00000000..233a2e6e --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/start.sh @@ -0,0 +1 @@ +node /vscode-tunnel/scripts/init-or-start.js diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/status.sh b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/status.sh new file mode 100644 index 00000000..4d8b473b --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/status.sh @@ -0,0 +1 @@ +code tunnel status diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/stop.sh b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/stop.sh new file mode 100644 index 00000000..35bca6ab --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/stop.sh @@ -0,0 +1 @@ +code tunnel kill diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/tunnel.sh b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/tunnel.sh new file mode 100644 index 00000000..996e7239 --- /dev/null +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/vscode-tunnel/scripts/tunnel.sh @@ -0,0 +1 @@ +code tunnel From 3ba4fbdf075e9f8e5a5dc55a20f843f09afb7ded Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 13:22:35 +0530 Subject: [PATCH 14/22] feat: added dwh connector for python-base-3.8.18 --- .../2.1.1-base-python-3.8.18/Dockerfile | 40 ++++++++++++++----- .../2.1.1-base-python-3.8.18/requirements.txt | 17 ++++++-- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile index d487f1a3..21f36917 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/Dockerfile @@ -5,14 +5,14 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # install system packages RUN apt-get update \ - # installation requirements + # Installation requirements && apt-get install -y --no-install-recommends \ apt-transport-https \ ca-certificates \ software-properties-common \ curl \ gnupg2 \ - # docker + # Set up Docker repository && mkdir -p /etc/apt/keyrings \ && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ && echo \ @@ -31,7 +31,7 @@ RUN apt-get update \ vim \ fish \ zsh \ - r-base=4.2.2.20221110-2 \ + r-base=4.2.2.20221110-2 \ r-recommended=4.2.2.20221110-2 \ build-essential \ sudo \ @@ -56,15 +56,33 @@ RUN apt-get update \ libxi6 \ openssl \ unzip \ + libpq-dev \ + libssl-dev \ + libffi-dev \ && rm -rf /var/lib/apt/lists/* -# install AWS CLI v2 +# Install AWS CLI v2 RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ && unzip -q awscliv2.zip \ && ./aws/install \ && rm -rf aws awscliv2.zip -# Needed for jupyterhub +# Install pyenv globally +RUN curl https://pyenv.run | bash \ + && mkdir -p /usr/local/pyenv \ + && mv /root/.pyenv/* /usr/local/pyenv \ + && rm -rf /root/.pyenv \ + && echo 'export PYENV_ROOT="/usr/local/pyenv"' >> /etc/profile.d/pyenv.sh \ + && echo 'export PATH="/usr/local/pyenv/bin:$PATH"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv init --path)"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile.d/pyenv.sh \ + && chmod -R 755 /usr/local/pyenv + +# Ensure pyenv is available to all users +ENV PYENV_ROOT=/usr/local/pyenv +ENV PATH="/usr/local/pyenv/bin:$PATH" + +# Needed for JupyterHub RUN npm install -g configurable-http-proxy \ && rm -rf ~/.npm @@ -74,9 +92,8 @@ COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir --upgrade \ pip \ setuptools \ - wheel - -RUN pip install -r /tmp/requirements.txt + wheel \ + && pip install -r /tmp/requirements.txt # SPARK: copy over spark config COPY ./spark /usr/local/spark @@ -87,7 +104,7 @@ RUN curl -fsSL https://code-server.dev/install.sh \ | sh -s -- --version 4.16.1 \ && pip install --no-cache-dir ./codeserver-proxy -# vscode cli +# VS Code CLI RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ && tar -xf vscode_cli.tar.gz \ && cp ./code /usr/local/bin \ @@ -102,7 +119,7 @@ RUN mkdir /vscode-tunnel/logs \ RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ - && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status COPY TUNNELING.md /vscode-tunnel/docs/README.md @@ -118,9 +135,10 @@ RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ && chmod -R a=rwx /usr/local/share/projector \ && pip install --no-cache-dir ./projector-proxy -# force shell to bash as Peak Platform isn't respecting jupyterhub option +# Force shell to bash as Peak Platform isn't respecting jupyterhub option RUN ln -sf /bin/bash /bin/sh +# Expose port and set the default command EXPOSE 8000 CMD ["jupyterhub", \ "--Spawner.default_url=/lab", \ diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt index e758edd4..31a10fad 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt @@ -16,6 +16,17 @@ pyls-isort==0.2.2 pyls-mypy==0.1.8 python-lsp-black==1.3.0 python-lsp-server[all]==1.4.1 -virtualenv==20.23.0 -peak-sdk==1.0.0 -jupysql==0.10.1 \ No newline at end of file +virtualenv==20.27.0 +peak-sdk==1.13.0 +jupysql==0.10.16 +psycopg2-binary==2.9.10 +redshift-connector==2.1.3 +snowflake-connector-python==3.12.3 +snowflake-sqlalchemy==1.6.1 +snowflake-snowpark-python==1.25.0 +pandas==2.2.3 +pyarrow==18.0.0 +boto3==1.35.65 +python-dotenv==1.0.1 +poetry==1.8.4 +sqlalchemy-redshift==0.8.14 From 8b48db00d08a39c1e3cb2bbd4465834023a56749 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 13:33:08 +0530 Subject: [PATCH 15/22] feat: fixed dokcerfile for python-base-3.11.5 --- .../ds-pack/2.1.1-base-python-3.11.5/Dockerfile | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile index f6163f40..253614be 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/Dockerfile @@ -1,4 +1,3 @@ -# Original content left intact FROM python:3.11.5-slim-bookworm # Set shell to be bash and fail any pipe if any step in the pipe fails @@ -6,6 +5,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # install system packages RUN apt-get update \ + # Installation requirements && apt-get install -y --no-install-recommends \ apt-transport-https \ ca-certificates \ @@ -24,6 +24,8 @@ RUN apt-get update \ && curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ && apt-get update \ && apt-get install -y --no-install-recommends \ + # Install system packages + # jq is needed for the Peak Platform jq \ nano \ vim \ @@ -80,15 +82,6 @@ RUN curl https://pyenv.run | bash \ ENV PYENV_ROOT=/usr/local/pyenv ENV PATH="/usr/local/pyenv/bin:$PATH" -# Install Poetry globally -RUN curl -sSL https://install.python-poetry.org | python3 - \ - && mv /root/.local/share/pypoetry /usr/local/share/pypoetry \ - && ln -s /usr/local/share/pypoetry/venv/bin/poetry /usr/local/bin/poetry \ - && chmod -R 755 /usr/local/share/pypoetry \ - && echo 'export PATH="/usr/local/bin:$PATH"' > /etc/profile.d/poetry.sh \ - && chmod +x /etc/profile.d/poetry.sh - - # Needed for JupyterHub RUN npm install -g configurable-http-proxy \ && rm -rf ~/.npm From 806a08708efc0a7ef783f9f7f3851a748c47202e Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 14:03:05 +0530 Subject: [PATCH 16/22] fix: version downgrade --- .../2.1.1-base-python-3.8.18/README.md | 20 +++++++++++++++---- .../2.1.1-base-python-3.8.18/requirements.txt | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md index e85e7773..86bc74e3 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md @@ -1,4 +1,4 @@ -# workspace-python-ds-pack-2.1.0-base-python-3.8.18 +# workspace-python-ds-pack-2.1.1-base-python-3.8.18 This is the Python Data Science pack image made for Workspaces. The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). @@ -37,6 +37,7 @@ vscode-cli 1.82.2 ### Python libraries installed ``` ipywidgets 8.1.0 +jupysql 0.10.16 jupyter-server-proxy 4.0.0 jupyterhub 4.0.2 jupyterlab-git 0.42.0 @@ -48,15 +49,26 @@ lckr-jupyterlab-variableinspector 3.0.9 mypy-ls 0.5.1 nbconvert 7.8.0 notebook 6.5.5 +peak-sdk 1.13.0 +poetry 1.8.4 +pyenv 0.10.16 pyls-black 0.4.7 pyls-flake 80.4.0 pyls-isort 0.2.2 pyls-mypy 0.1.8 +python-dotenv 1.0.1 python-lsp-black 1.3.0 python-lsp-server[all] 1.4.1 -virtualenv 20.24.5 -peak-sdk 1.0.0 -jupysql 0.10.1 +virtualenv 20.27.0 +``` + +### DWH connector libraries installed +``` +redshift-connector 2.1.3 +sqlalchemy-redshift 0.8.14 +snowflake-connector-python 3.12.3 +snowflake-sqlalchemy 1.6.1 +snowflake-snowpark-python 1.25.0 ``` ### VS Code extensions installed diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt index 31a10fad..84a63b93 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt @@ -24,7 +24,7 @@ redshift-connector==2.1.3 snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 -pandas==2.2.3 +pandas==2.0.3 pyarrow==18.0.0 boto3==1.35.65 python-dotenv==1.0.1 From 7f73a09981297a7b68ae1e0f007867b34194555d Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 14:20:53 +0530 Subject: [PATCH 17/22] fix: pyarrow version --- .../python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt index 84a63b93..2368ac9b 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt @@ -25,7 +25,7 @@ snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 pandas==2.0.3 -pyarrow==18.0.0 +pyarrow==17.0.0 boto3==1.35.65 python-dotenv==1.0.1 poetry==1.8.4 From 1016019a12b3cfda45c1edb460061cedc68e9f53 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 14:35:07 +0530 Subject: [PATCH 18/22] fix: updated dependencies --- .../python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt | 1 - .../python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt index 31a10fad..3037611b 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/requirements.txt @@ -25,7 +25,6 @@ snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 pandas==2.2.3 -pyarrow==18.0.0 boto3==1.35.65 python-dotenv==1.0.1 poetry==1.8.4 diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt index 2368ac9b..d695fd1e 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/requirements.txt @@ -18,14 +18,13 @@ python-lsp-black==1.3.0 python-lsp-server[all]==1.4.1 virtualenv==20.27.0 peak-sdk==1.13.0 -jupysql==0.10.16 +jupysql==0.10.1 psycopg2-binary==2.9.10 redshift-connector==2.1.3 snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 pandas==2.0.3 -pyarrow==17.0.0 boto3==1.35.65 python-dotenv==1.0.1 poetry==1.8.4 From 858f7772725f5d9a1c00855ad88d01c2c5f56ee2 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 15:03:24 +0530 Subject: [PATCH 19/22] fixed readme --- .../2.1.1-base-python-3.11.5/README.md | 24 +++++++++++++++---- .../2.1.1-base-python-3.8.18/README.md | 9 ++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md index a5fbb0d0..d38a483f 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md +++ b/workspace/python/ds-pack/2.1.1-base-python-3.11.5/README.md @@ -1,4 +1,4 @@ -# workspace-python-ds-pack-2.1.0-base-python-3.11.5 +# workspace-python-ds-pack-2.1.1-base-python-3.11.5 This is the Python Data Science pack image made for Workspaces. The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). @@ -43,27 +43,43 @@ vscode-cli 1.82.2 ### Python libraries installed ``` +boto3 1.35.65 ipywidgets 8.1.0 jupyter-server-proxy 4.0.0 jupyterhub 4.0.2 +jupyterlab 3.6.5 jupyterlab-git 0.42.0 jupyterlab-lsp 4.2.0 -jupyterlab 3.6.5 jupyterlab_widgets 3.0.8 +jupysql 0.10.16 jupytext 1.15.1 lckr-jupyterlab-variableinspector 3.0.9 mypy-ls 0.5.1 nbconvert 7.8.0 notebook 6.5.5 +pandas 2.2.3 +peak-sdk 1.13.0 +poetry 1.8.4 +pyenv 2.4.19 pyls-black 0.4.7 pyls-flake 80.4.0 pyls-isort 0.2.2 pyls-mypy 0.1.8 +python-dotenv 1.0.1 python-lsp-black 1.3.0 python-lsp-server[all] 1.4.1 virtualenv 20.24.5 -peak-sdk 1.0.0 -jupysql 0.10.1 +``` + + +### DWH connector libraries installed +``` +psycopg2-binary 2.9.10 +redshift-connector 2.1.3 +sqlalchemy-redshift 0.8.14 +snowflake-connector-python 3.12.3 +snowflake-sqlalchemy 1.6.1 +snowflake-snowpark-python 1.25.0 ``` ### VS Code Tunneling diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md index 86bc74e3..d670e909 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md +++ b/workspace/python/ds-pack/2.1.1-base-python-3.8.18/README.md @@ -36,22 +36,24 @@ vscode-cli 1.82.2 ### Python libraries installed ``` +boto3 1.35.65 ipywidgets 8.1.0 -jupysql 0.10.16 jupyter-server-proxy 4.0.0 jupyterhub 4.0.2 +jupyterlab 3.6.5 jupyterlab-git 0.42.0 jupyterlab-lsp 4.2.0 -jupyterlab 3.6.5 jupyterlab_widgets 3.0.8 +jupysql 0.10.1 jupytext 1.15.1 lckr-jupyterlab-variableinspector 3.0.9 mypy-ls 0.5.1 nbconvert 7.8.0 notebook 6.5.5 +pandas 2.2.3 peak-sdk 1.13.0 poetry 1.8.4 -pyenv 0.10.16 +pyenv 2.4.19 pyls-black 0.4.7 pyls-flake 80.4.0 pyls-isort 0.2.2 @@ -64,6 +66,7 @@ virtualenv 20.27.0 ### DWH connector libraries installed ``` +psycopg2-binary 2.9.10 redshift-connector 2.1.3 sqlalchemy-redshift 0.8.14 snowflake-connector-python 3.12.3 From d75ebe7303e43496a93e17795b9bcef6e4b5efc2 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 15:25:00 +0530 Subject: [PATCH 20/22] feat: added dependencies for python-base 3.9 & 3.10 --- .../2.1.1-base-python-3.10.13/Dockerfile | 40 ++++++++++++++----- .../requirements.txt | 16 ++++++-- .../2.1.1-base-python-3.9.18/Dockerfile | 40 ++++++++++++++----- .../2.1.1-base-python-3.9.18/requirements.txt | 16 ++++++-- 4 files changed, 84 insertions(+), 28 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile index 0cf35eb6..d1998d99 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/Dockerfile @@ -5,14 +5,14 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # install system packages RUN apt-get update \ - # installation requirements + # Installation requirements && apt-get install -y --no-install-recommends \ apt-transport-https \ ca-certificates \ software-properties-common \ curl \ gnupg2 \ - # docker + # Set up Docker repository && mkdir -p /etc/apt/keyrings \ && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ && echo \ @@ -56,15 +56,33 @@ RUN apt-get update \ libxi6 \ openssl \ unzip \ + libpq-dev \ + libssl-dev \ + libffi-dev \ && rm -rf /var/lib/apt/lists/* -# install AWS CLI v2 +# Install AWS CLI v2 RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ && unzip -q awscliv2.zip \ && ./aws/install \ && rm -rf aws awscliv2.zip -# Needed for jupyterhub +# Install pyenv globally +RUN curl https://pyenv.run | bash \ + && mkdir -p /usr/local/pyenv \ + && mv /root/.pyenv/* /usr/local/pyenv \ + && rm -rf /root/.pyenv \ + && echo 'export PYENV_ROOT="/usr/local/pyenv"' >> /etc/profile.d/pyenv.sh \ + && echo 'export PATH="/usr/local/pyenv/bin:$PATH"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv init --path)"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile.d/pyenv.sh \ + && chmod -R 755 /usr/local/pyenv + +# Ensure pyenv is available to all users +ENV PYENV_ROOT=/usr/local/pyenv +ENV PATH="/usr/local/pyenv/bin:$PATH" + +# Needed for JupyterHub RUN npm install -g configurable-http-proxy \ && rm -rf ~/.npm @@ -74,9 +92,8 @@ COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir --upgrade \ pip \ setuptools \ - wheel - -RUN pip install -r /tmp/requirements.txt + wheel \ + && pip install -r /tmp/requirements.txt # SPARK: copy over spark config COPY ./spark /usr/local/spark @@ -87,7 +104,7 @@ RUN curl -fsSL https://code-server.dev/install.sh \ | sh -s -- --version 4.16.1 \ && pip install --no-cache-dir ./codeserver-proxy -# vscode cli +# VS Code CLI RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ && tar -xf vscode_cli.tar.gz \ && cp ./code /usr/local/bin \ @@ -102,11 +119,11 @@ RUN mkdir /vscode-tunnel/logs \ RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ - && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status COPY TUNNELING.md /vscode-tunnel/docs/README.md - + RUN git config --global credential.helper store \ && git config --global core.filemode false @@ -118,9 +135,10 @@ RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ && chmod -R a=rwx /usr/local/share/projector \ && pip install --no-cache-dir ./projector-proxy -# force shell to bash as Peak Platform isn't respecting jupyterhub option +# Force shell to bash as Peak Platform isn't respecting jupyterhub option RUN ln -sf /bin/bash /bin/sh +# Expose port and set the default command EXPOSE 8000 CMD ["jupyterhub", \ "--Spawner.default_url=/lab", \ diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt index e758edd4..d695fd1e 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt @@ -16,6 +16,16 @@ pyls-isort==0.2.2 pyls-mypy==0.1.8 python-lsp-black==1.3.0 python-lsp-server[all]==1.4.1 -virtualenv==20.23.0 -peak-sdk==1.0.0 -jupysql==0.10.1 \ No newline at end of file +virtualenv==20.27.0 +peak-sdk==1.13.0 +jupysql==0.10.1 +psycopg2-binary==2.9.10 +redshift-connector==2.1.3 +snowflake-connector-python==3.12.3 +snowflake-sqlalchemy==1.6.1 +snowflake-snowpark-python==1.25.0 +pandas==2.0.3 +boto3==1.35.65 +python-dotenv==1.0.1 +poetry==1.8.4 +sqlalchemy-redshift==0.8.14 diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile index b527e491..10ba8b40 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/Dockerfile @@ -5,14 +5,14 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # install system packages RUN apt-get update \ - # installation requirements + # Installation requirements && apt-get install -y --no-install-recommends \ apt-transport-https \ ca-certificates \ software-properties-common \ curl \ gnupg2 \ - # docker + # Set up Docker repository && mkdir -p /etc/apt/keyrings \ && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ && echo \ @@ -56,15 +56,33 @@ RUN apt-get update \ libxi6 \ openssl \ unzip \ + libpq-dev \ + libssl-dev \ + libffi-dev \ && rm -rf /var/lib/apt/lists/* -# install AWS CLI v2 +# Install AWS CLI v2 RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.7.4.zip" -o "awscliv2.zip" \ && unzip -q awscliv2.zip \ && ./aws/install \ && rm -rf aws awscliv2.zip -# Needed for jupyterhub +# Install pyenv globally +RUN curl https://pyenv.run | bash \ + && mkdir -p /usr/local/pyenv \ + && mv /root/.pyenv/* /usr/local/pyenv \ + && rm -rf /root/.pyenv \ + && echo 'export PYENV_ROOT="/usr/local/pyenv"' >> /etc/profile.d/pyenv.sh \ + && echo 'export PATH="/usr/local/pyenv/bin:$PATH"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv init --path)"' >> /etc/profile.d/pyenv.sh \ + && echo 'eval "$(pyenv virtualenv-init -)"' >> /etc/profile.d/pyenv.sh \ + && chmod -R 755 /usr/local/pyenv + +# Ensure pyenv is available to all users +ENV PYENV_ROOT=/usr/local/pyenv +ENV PATH="/usr/local/pyenv/bin:$PATH" + +# Needed for JupyterHub RUN npm install -g configurable-http-proxy \ && rm -rf ~/.npm @@ -74,9 +92,8 @@ COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir --upgrade \ pip \ setuptools \ - wheel - -RUN pip install -r /tmp/requirements.txt + wheel \ + && pip install -r /tmp/requirements.txt # SPARK: copy over spark config COPY ./spark /usr/local/spark @@ -87,7 +104,7 @@ RUN curl -fsSL https://code-server.dev/install.sh \ | sh -s -- --version 4.16.1 \ && pip install --no-cache-dir ./codeserver-proxy -# vscode cli +# VS Code CLI RUN curl -Lk 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' --output vscode_cli.tar.gz \ && tar -xf vscode_cli.tar.gz \ && cp ./code /usr/local/bin \ @@ -102,11 +119,11 @@ RUN mkdir /vscode-tunnel/logs \ RUN cp vscode-tunnel/scripts/start.sh /usr/bin/start-tunnel \ && cp vscode-tunnel/scripts/stop.sh /usr/bin/stop-tunnel \ && cp vscode-tunnel/scripts/refresh.sh /usr/bin/restart-tunnel \ - && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ + && cp vscode-tunnel/scripts/status.sh /usr/bin/tunnel-status \ && chmod +x /usr/bin/start-tunnel /usr/bin/stop-tunnel /usr/bin/restart-tunnel /usr/bin/tunnel-status COPY TUNNELING.md /vscode-tunnel/docs/README.md - + RUN git config --global credential.helper store \ && git config --global core.filemode false @@ -118,9 +135,10 @@ RUN pip install --no-cache-dir projector-installer==1.7.0 --upgrade \ && chmod -R a=rwx /usr/local/share/projector \ && pip install --no-cache-dir ./projector-proxy -# force shell to bash as Peak Platform isn't respecting jupyterhub option +# Force shell to bash as Peak Platform isn't respecting jupyterhub option RUN ln -sf /bin/bash /bin/sh +# Expose port and set the default command EXPOSE 8000 CMD ["jupyterhub", \ "--Spawner.default_url=/lab", \ diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt index e758edd4..d695fd1e 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt @@ -16,6 +16,16 @@ pyls-isort==0.2.2 pyls-mypy==0.1.8 python-lsp-black==1.3.0 python-lsp-server[all]==1.4.1 -virtualenv==20.23.0 -peak-sdk==1.0.0 -jupysql==0.10.1 \ No newline at end of file +virtualenv==20.27.0 +peak-sdk==1.13.0 +jupysql==0.10.1 +psycopg2-binary==2.9.10 +redshift-connector==2.1.3 +snowflake-connector-python==3.12.3 +snowflake-sqlalchemy==1.6.1 +snowflake-snowpark-python==1.25.0 +pandas==2.0.3 +boto3==1.35.65 +python-dotenv==1.0.1 +poetry==1.8.4 +sqlalchemy-redshift==0.8.14 From ebf88be5ecda04aae0e77d6bd3c1f9f428ea0dc7 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Thu, 21 Nov 2024 15:34:27 +0530 Subject: [PATCH 21/22] chore: updated readme --- .../2.1.1-base-python-3.10.13/README.md | 25 +++++++++++++++---- .../2.1.1-base-python-3.9.18/README.md | 25 +++++++++++++++---- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md index 302ba30a..d37061f9 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/README.md @@ -1,4 +1,4 @@ -# workspace-python-ds-pack-2.1.0-base-python-3.10.13 +# workspace-python-ds-pack-2.1.1-base-python-3.10.13 This is the Python Data Science pack image made for Workspaces. The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). @@ -36,27 +36,42 @@ vscode-cli 1.82.2 ### Python libraries installed ``` +boto3 1.35.65 ipywidgets 8.1.0 jupyter-server-proxy 4.0.0 jupyterhub 4.0.2 +jupyterlab 3.6.5 jupyterlab-git 0.42.0 jupyterlab-lsp 4.2.0 -jupyterlab 3.6.5 jupyterlab_widgets 3.0.8 +jupysql 0.10.1 jupytext 1.15.1 lckr-jupyterlab-variableinspector 3.0.9 mypy-ls 0.5.1 nbconvert 7.8.0 notebook 6.5.5 +pandas 2.2.3 +peak-sdk 1.13.0 +poetry 1.8.4 +pyenv 2.4.19 pyls-black 0.4.7 pyls-flake 80.4.0 pyls-isort 0.2.2 pyls-mypy 0.1.8 +python-dotenv 1.0.1 python-lsp-black 1.3.0 python-lsp-server[all] 1.4.1 -virtualenv 20.24.5 -peak-sdk 1.0.0 -jupysql 0.10.1 +virtualenv 20.27.0 +``` + +### DWH connector libraries installed +``` +psycopg2-binary 2.9.10 +redshift-connector 2.1.3 +sqlalchemy-redshift 0.8.14 +snowflake-connector-python 3.12.3 +snowflake-sqlalchemy 1.6.1 +snowflake-snowpark-python 1.25.0 ``` ### VS Code extensions installed diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md index c654d1b2..dcd51a50 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/README.md @@ -1,4 +1,4 @@ -# workspace-python-ds-pack-2.1.0-base-python-3.9.18 +# workspace-python-ds-pack-2.1.1-base-python-3.9.18 This is the Python Data Science pack image made for Workspaces. The image runs a [Jupyterhub](https://jupyter.org/hub) server that has [PyCharm](https://lp.jetbrains.com/projector/) and [VSCode](https://github.com/coder/code-server) pre-installed with all of the essential and most used packages. The images also comes pre installed with [peak-sdk](https://docs.peak.ai/sdk/). @@ -36,27 +36,42 @@ vscode-cli 1.82.2 ### Python libraries installed ``` +boto3 1.35.65 ipywidgets 8.1.0 jupyter-server-proxy 4.0.0 jupyterhub 4.0.2 +jupyterlab 3.6.5 jupyterlab-git 0.42.0 jupyterlab-lsp 4.2.0 -jupyterlab 3.6.5 jupyterlab_widgets 3.0.8 +jupysql 0.10.1 jupytext 1.15.1 lckr-jupyterlab-variableinspector 3.0.9 mypy-ls 0.5.1 nbconvert 7.8.0 notebook 6.5.5 +pandas 2.2.3 +peak-sdk 1.13.0 +poetry 1.8.4 +pyenv 2.4.19 pyls-black 0.4.7 pyls-flake 80.4.0 pyls-isort 0.2.2 pyls-mypy 0.1.8 +python-dotenv 1.0.1 python-lsp-black 1.3.0 python-lsp-server[all] 1.4.1 -virtualenv 20.24.5 -peak-sdk 1.0.0 -jupysql 0.10.1 +virtualenv 20.27.0 +``` + +### DWH connector libraries installed +``` +psycopg2-binary 2.9.10 +redshift-connector 2.1.3 +sqlalchemy-redshift 0.8.14 +snowflake-connector-python 3.12.3 +snowflake-sqlalchemy 1.6.1 +snowflake-snowpark-python 1.25.0 ``` ### VS Code extensions installed From 285c110c0f05ef639330ccbbc245b095d3bda241 Mon Sep 17 00:00:00 2001 From: "vikash.ranjan" Date: Fri, 22 Nov 2024 15:08:00 +0530 Subject: [PATCH 22/22] fix: pands version --- .../python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt | 2 +- .../python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt index d695fd1e..c6390064 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.10.13/requirements.txt @@ -24,7 +24,7 @@ redshift-connector==2.1.3 snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 -pandas==2.0.3 +pandas==2.2.3 boto3==1.35.65 python-dotenv==1.0.1 poetry==1.8.4 diff --git a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt index d695fd1e..c6390064 100644 --- a/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt +++ b/workspace/python/ds-pack/2.1.1-base-python-3.9.18/requirements.txt @@ -24,7 +24,7 @@ redshift-connector==2.1.3 snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 snowflake-snowpark-python==1.25.0 -pandas==2.0.3 +pandas==2.2.3 boto3==1.35.65 python-dotenv==1.0.1 poetry==1.8.4